wip(mobile): in-progress mobile + voice polish

Snapshot of in-progress work parked here so group-DM-polish can land
cleanly on main. Touches MainContent + mobile-ui.md which overlap with
group-DM-polish; rebase onto post-merge main and resolve conflicts on
those two files manually.

Files: MainContent, MessageInput, MobileVoiceFullScreen (+test), StreamTile,
VoiceUser, useLiveKit, useVisualViewportInset, AudioManager, voiceStore,
voice utils, mobile-ui.md, voice.md, mobile-parity handoff doc.
This commit is contained in:
Jannis Braun
2026-05-10 21:26:40 +02:00
parent 6fb38d391c
commit dbb9b2c34b
13 changed files with 1241 additions and 232 deletions
+33 -2
View File
@@ -210,6 +210,7 @@ export function useLiveKit() {
const inputVolume = useVoiceStore((s) => s.inputVolume);
const inputDeviceId = useVoiceStore((s) => s.inputDeviceId);
const cameraDeviceId = useVoiceStore((s) => s.cameraDeviceId);
const micPermissionDenied = useVoiceStore((s) => s.micPermissionDenied);
const echoCancellation = useVoiceStore((s) => s.echoCancellation);
const noiseSuppression = useVoiceStore((s) => s.noiseSuppression);
const autoGainControl = useVoiceStore((s) => s.autoGainControl);
@@ -388,6 +389,19 @@ export function useLiveKit() {
const micPub = r.localParticipant.getTrackPublications()
.find(p => p.source === Track.Source.Microphone);
// Mic permission denied — user joined as a listener. Tear down any
// stale publication (defensive: should not exist on first join, but
// covers the edge where permission was revoked mid-call) and skip
// the publish branch entirely. Re-attempt is gated behind the
// `requestMicPermission` user-gesture path which clears the flag
// and triggers this effect to re-run via its dep.
if (micPermissionDenied) {
if (micPub?.track) {
await r.localParticipant.unpublishTrack(micPub.track as LocalAudioTrack).catch(() => {});
}
return;
}
// If effectively muted or deafened, mute the track in-place (keep it published)
if (effectiveMuted || effectiveDeafened) {
if (micPub?.track && !micPub.isMuted) {
@@ -399,7 +413,24 @@ export function useLiveKit() {
// Not muted — ensure the AudioManager pipeline is on the right device
// and at the right volume, then republish if the published track is
// stale or missing.
await audioManager.setInputDevice(inputDeviceId);
try {
await audioManager.setInputDevice(inputDeviceId);
} catch (err: any) {
// Late denial (e.g. iOS standalone PWA where the pre-arm in
// `joinVoiceChannel` ran inside the gesture but the prompt is
// delivered out-of-band; the user denies after `room.connect()`
// has already succeeded). Promote to the listener-mode flag so
// we don't loop forever trying to re-acquire.
if (err?.name === 'NotAllowedError') {
useVoiceStore.getState().setMicPermissionDenied(true);
useUIStore.getState().addToast(
'Microphone access denied. You joined as a listener — tap "Allow microphone" to grant access.',
'warning',
);
return;
}
throw err;
}
audioManager.setInputVolume(inputVolume);
await republishMicrophone(r, lastMicGenRef);
} catch (err) {
@@ -417,7 +448,7 @@ export function useLiveKit() {
return () => {
unsubscribeResume();
};
}, [isMuted, isDeafened, spaceMutedUserIds, spaceDeafenedUserIds, permissionMutedUserIds, inputDeviceId, inputVolume, isConnected, echoCancellation, noiseSuppression, autoGainControl, rnnoiseEnabled]);
}, [isMuted, isDeafened, spaceMutedUserIds, spaceDeafenedUserIds, permissionMutedUserIds, inputDeviceId, inputVolume, isConnected, echoCancellation, noiseSuppression, autoGainControl, rnnoiseEnabled, micPermissionDenied]);
// Subscribe to upstream-input-track-end events from AudioManager whenever a
// room is connected. The published mic track is a clone of a WebAudio
@@ -60,6 +60,16 @@ export interface VisualViewportInset {
value: string;
/** True if the soft keyboard is occluding the bottom of the layout viewport. */
keyboardOpen: boolean;
/**
* True if a text-entry element currently has focus. Used as a focus-based
* fallback signal for the iOS PWA case where `interactive-widget=resizes-content`
* (or iOS's native standalone behavior) shrinks the layout viewport itself
* for the keyboard — `vv.height` ends up matching `innerHeight`, so
* `keyboardOpen` (which infers from height delta) stays false even though
* the keyboard IS up. Consumers that need a "is the keyboard most likely
* up" signal should OR `keyboardOpen || textInputFocused`.
*/
textInputFocused: boolean;
/**
* Live `visualViewport.height` in pixels, or `null` if `visualViewport` is
* unavailable. Consumers that want to size a container to the visible
@@ -77,12 +87,16 @@ export interface VisualViewportInset {
const FALLBACK: VisualViewportInset = {
value: 'env(safe-area-inset-bottom)',
keyboardOpen: false,
textInputFocused: false,
height: null,
offsetTop: null,
};
export function useVisualViewportInset(): VisualViewportInset {
const [inset, setInset] = useState<VisualViewportInset>(FALLBACK);
// Mutable ref so `measure()` reads the latest focus state without React
// re-renders racing the visualViewport update path.
const textInputFocusedRef = { current: false } as { current: boolean };
useEffect(() => {
const vv = window.visualViewport;
@@ -104,12 +118,14 @@ export function useVisualViewportInset(): VisualViewportInset {
? {
value: `${Math.round(occlusion)}px`,
keyboardOpen: true,
textInputFocused: textInputFocusedRef.current,
height: vv.height,
offsetTop: vv.offsetTop,
}
: {
value: 'env(safe-area-inset-bottom)',
keyboardOpen: false,
textInputFocused: textInputFocusedRef.current,
height: vv.height,
offsetTop: vv.offsetTop,
};
@@ -117,9 +133,24 @@ export function useVisualViewportInset(): VisualViewportInset {
// Functional update + shallow compare so identical re-measurements
// don't churn React state every animation frame during keyboard
// transitions.
//
// ALL returned fields must be in this comparison, otherwise an
// observable change to one of them is silently dropped — the historical
// bug was `textInputFocused` being absent here. On iOS PWA standalone
// the layout viewport is shrunk natively for the keyboard, so
// `vv.height` matches `innerHeight` and `keyboardOpen`/`value` stay
// unchanged across the transition. The only signal that flips is
// `textInputFocused`. Without it in the compare, consumers
// (`MessageInput`'s `--composer-clearance` effect, the composer's
// inline `bottom` style) never see the focus change propagate, the
// composer stayed pinned above the home indicator while the keyboard
// was actually up, and on close the clearance variable was computed
// off a stale `bottom` value — surfacing as a -4 px overlap between
// the last message and the composer's top edge.
setInset((prev) =>
prev.value === next.value &&
prev.keyboardOpen === next.keyboardOpen &&
prev.textInputFocused === next.textInputFocused &&
prev.height === next.height &&
prev.offsetTop === next.offsetTop
? prev
@@ -187,6 +218,11 @@ export function useVisualViewportInset(): VisualViewportInset {
tag === 'TEXTAREA' ||
(t as HTMLElement).isContentEditable === true;
if (!editable) return;
// Track focus state — used as a fallback signal in the iOS PWA case
// where `vv.height` doesn't shrink for the keyboard (because iOS
// native-shifts the layout viewport instead). `focusin` → focused;
// `focusout` → unfocused. Capture phase listener so we see all events.
textInputFocusedRef.current = e.type === 'focusin';
// Immediate measure + a polling window for laggy iOS PWA event flows.
update();
startPolling(600);