feat(server): fan dm_call_end out on host ring timeout
This commit is contained in:
@@ -26,6 +26,7 @@ import { adminRoutes } from './routes/admin.js';
|
|||||||
import { gifRoutes } from './routes/gif.js';
|
import { gifRoutes } from './routes/gif.js';
|
||||||
import { federationRoutes } from './routes/federation.js';
|
import { federationRoutes } from './routes/federation.js';
|
||||||
import { startFederationWorkers, stopFederationWorkers } from './utils/federationWorker.js';
|
import { startFederationWorkers, stopFederationWorkers } from './utils/federationWorker.js';
|
||||||
|
import { registerCallRelayHooks } from './ws/events.js';
|
||||||
|
|
||||||
import { registerWebSocket } from './ws/handler.js';
|
import { registerWebSocket } from './ws/handler.js';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
@@ -124,6 +125,9 @@ async function main(): Promise<void> {
|
|||||||
// Log ffmpeg availability at startup (so admins see the warning immediately)
|
// Log ffmpeg availability at startup (so admins see the warning immediately)
|
||||||
checkFfmpeg();
|
checkFfmpeg();
|
||||||
|
|
||||||
|
// Register WS-layer call relay hooks (ring-timeout fan-out).
|
||||||
|
registerCallRelayHooks();
|
||||||
|
|
||||||
// Start federation background workers (outbox delivery, file download, health check)
|
// Start federation background workers (outbox delivery, file download, health check)
|
||||||
startFederationWorkers();
|
startFederationWorkers();
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import { getDb, getRawDb, schema } from '../db/index.js';
|
|||||||
import { config } from '../config.js';
|
import { config } from '../config.js';
|
||||||
import { connectionManager } from '../ws/handler.js';
|
import { connectionManager } from '../ws/handler.js';
|
||||||
import type { FederatedCallEntry, DmRoomMeta } from '../ws/handler.js';
|
import type { FederatedCallEntry, DmRoomMeta } from '../ws/handler.js';
|
||||||
import { mapCallReasonToEventReason, type CallFanoutFailure } from '../ws/events.js';
|
import { mapCallReasonToEventReason, type CallFanoutFailure } from '../utils/federationOutbox.js';
|
||||||
import type { DmCallUndeliverableFailure } from '@backspace/shared';
|
import type { DmCallUndeliverableFailure } from '@backspace/shared';
|
||||||
import { sanitizeUser } from '../utils/sanitize.js';
|
import { sanitizeUser } from '../utils/sanitize.js';
|
||||||
import { deleteAttachmentFiles, deleteUploadFile } from '../utils/fileCleanup.js';
|
import { deleteAttachmentFiles, deleteUploadFile } from '../utils/fileCleanup.js';
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import * as schema from '../db/schema.js';
|
|||||||
import { eq, and, inArray } from 'drizzle-orm';
|
import { eq, and, inArray } from 'drizzle-orm';
|
||||||
import { generateSnowflake } from './snowflake.js';
|
import { generateSnowflake } from './snowflake.js';
|
||||||
import crypto from 'node:crypto';
|
import crypto from 'node:crypto';
|
||||||
import type { FederationRelayEvent, FederationRelayParticipant, FederationRelayAttachment, DmMessageWithUser, FederationRelayRequest } from '@backspace/shared';
|
import type { FederationRelayEvent, FederationRelayParticipant, FederationRelayAttachment, DmMessageWithUser, FederationRelayRequest, DmCallUndeliverableReason } from '@backspace/shared';
|
||||||
import { getOurOrigin, buildFederationHeaders, generateHmacSecret } from './federationAuth.js';
|
import { getOurOrigin, buildFederationHeaders, generateHmacSecret } from './federationAuth.js';
|
||||||
import { extractDomain } from '../routes/federation.js';
|
import { extractDomain } from '../routes/federation.js';
|
||||||
import { racePeering, ensurePeered } from './federationPeering.js';
|
import { racePeering, ensurePeered } from './federationPeering.js';
|
||||||
@@ -527,6 +527,23 @@ export type CallRelayResult =
|
|||||||
| { ok: true }
|
| { ok: true }
|
||||||
| { ok: false; reason: CallRelayFailureReason; error: string };
|
| { ok: false; reason: CallRelayFailureReason; error: string };
|
||||||
|
|
||||||
|
/** Per-peer failure record returned by Path-1 call fan-out helpers. */
|
||||||
|
export interface CallFanoutFailure {
|
||||||
|
origin: string;
|
||||||
|
peerLabel?: string;
|
||||||
|
reason: DmCallUndeliverableReason;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Map a sendCallRelay reason to the dm_call_undeliverable event-surface reason. */
|
||||||
|
export function mapCallReasonToEventReason(reason: CallRelayFailureReason): DmCallUndeliverableReason {
|
||||||
|
switch (reason) {
|
||||||
|
case 'peer_rejected': return 'peer_rejected';
|
||||||
|
case 'peer_awaiting_approval': return 'peer_awaiting_approval';
|
||||||
|
case 'peer_transient_failure': return 'peer_transient_failure';
|
||||||
|
case 'post_failed': return 'peer_transient_failure'; // 4xx looks transient to users
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Send call signaling events directly to a remote peer (bypasses outbox).
|
* Send call signaling events directly to a remote peer (bypasses outbox).
|
||||||
* Latency-sensitive: if no active peer exists, race an ensurePeered handshake
|
* Latency-sensitive: if no active peer exists, race an ensurePeered handshake
|
||||||
|
|||||||
@@ -205,6 +205,34 @@ describe('handleDmCallReject Path-2 relay failure', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('ring-timeout fan-out hook', () => {
|
||||||
|
it('invokes the fan-out hook when the ring timer fires', async () => {
|
||||||
|
const connectionManager = await importManager();
|
||||||
|
|
||||||
|
const hook = vi.fn(async () => {});
|
||||||
|
connectionManager.setRingTimeoutFanoutHook(hook);
|
||||||
|
|
||||||
|
vi.useFakeTimers();
|
||||||
|
try {
|
||||||
|
connectionManager.createDmRoom('dm-ringout-test', 'caller-ringout');
|
||||||
|
vi.advanceTimersByTime(60_000 + 10);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(hook).toHaveBeenCalledWith('dm-ringout-test', 'caller-ringout');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('registerCallRelayHooks wires up the hook', async () => {
|
||||||
|
const connectionManager = await importManager();
|
||||||
|
const { registerCallRelayHooks } = await importSUT();
|
||||||
|
|
||||||
|
const setSpy = vi.spyOn(connectionManager, 'setRingTimeoutFanoutHook');
|
||||||
|
registerCallRelayHooks();
|
||||||
|
expect(setSpy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('handleDmCallAccept Path-2 relay failure', () => {
|
describe('handleDmCallAccept Path-2 relay failure', () => {
|
||||||
it('emits dm_call_undeliverable { phase:"accept", terminal:true } and clears the fedCall when the relay fails', async () => {
|
it('emits dm_call_undeliverable { phase:"accept", terminal:true } and clears the fedCall when the relay fails', async () => {
|
||||||
const { handleDmCallAcceptForTest } = await importSUT();
|
const { handleDmCallAcceptForTest } = await importSUT();
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import type { VoiceRoom, DmRoomMeta, SpaceRoomMeta } from './handler.js';
|
|||||||
import { isMember, getChannelSpaceId, isDmMember, hasPermission, computePermissions, PermissionBits } from '../utils/permissions.js';
|
import { isMember, getChannelSpaceId, isDmMember, hasPermission, computePermissions, PermissionBits } from '../utils/permissions.js';
|
||||||
import { broadcastDmMessage, getDmMessageWithUser } from '../routes/dm.js';
|
import { broadcastDmMessage, getDmMessageWithUser } from '../routes/dm.js';
|
||||||
import { MAX_MESSAGE_LENGTH, type MessageWithUser, type Attachment, type DmMessageWithUser, type Embed, type Activity, type ActivityType, type ActivityTimestamps, type ActivityAssets, type ServerEvent, type DmCallUndeliverableFailure, type DmCallUndeliverableReason } from '@backspace/shared';
|
import { MAX_MESSAGE_LENGTH, type MessageWithUser, type Attachment, type DmMessageWithUser, type Embed, type Activity, type ActivityType, type ActivityTimestamps, type ActivityAssets, type ServerEvent, type DmCallUndeliverableFailure, type DmCallUndeliverableReason } from '@backspace/shared';
|
||||||
import type { CallRelayFailureReason, CallRelayResult } from '../utils/federationOutbox.js';
|
import type { CallRelayResult, CallFanoutFailure } from '../utils/federationOutbox.js';
|
||||||
|
import { mapCallReasonToEventReason } from '../utils/federationOutbox.js';
|
||||||
import { ACTIVITY_LIMITS } from '@backspace/shared/src/activities.js';
|
import { ACTIVITY_LIMITS } from '@backspace/shared/src/activities.js';
|
||||||
import { sanitizeUser } from '../utils/sanitize.js';
|
import { sanitizeUser } from '../utils/sanitize.js';
|
||||||
import { deleteAttachmentFiles } from '../utils/fileCleanup.js';
|
import { deleteAttachmentFiles } from '../utils/fileCleanup.js';
|
||||||
@@ -1971,16 +1972,6 @@ async function sendFederatedCallStart(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Map a sendCallRelay reason to the event-surface reason. */
|
|
||||||
export function mapCallReasonToEventReason(reason: CallRelayFailureReason): DmCallUndeliverableReason {
|
|
||||||
switch (reason) {
|
|
||||||
case 'peer_rejected': return 'peer_rejected';
|
|
||||||
case 'peer_awaiting_approval': return 'peer_awaiting_approval';
|
|
||||||
case 'peer_transient_failure': return 'peer_transient_failure';
|
|
||||||
case 'post_failed': return 'peer_transient_failure'; // 4xx looks transient to users
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Build a DmCallUndeliverableFailure from a failed CallRelayResult, enriching with peer label. */
|
/** Build a DmCallUndeliverableFailure from a failed CallRelayResult, enriching with peer label. */
|
||||||
function buildFailureFromResult(
|
function buildFailureFromResult(
|
||||||
result: Extract<CallRelayResult, { ok: false }>,
|
result: Extract<CallRelayResult, { ok: false }>,
|
||||||
@@ -1998,12 +1989,6 @@ function buildFailureFromResult(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Per-peer failure record returned by Path-1 fan-out helpers. */
|
|
||||||
export interface CallFanoutFailure {
|
|
||||||
origin: string;
|
|
||||||
peerLabel?: string;
|
|
||||||
reason: DmCallUndeliverableReason;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Emit dm_call_undeliverable to the caller. If terminal, also destroy the
|
* Emit dm_call_undeliverable to the caller. If terminal, also destroy the
|
||||||
@@ -2474,6 +2459,21 @@ function handleVoiceDisconnect(event: Record<string, unknown>, userId: string):
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register the ring-timeout fan-out so a host-side 60s auto-clean notifies remote peers.
|
||||||
|
* Called from server startup; split from module-load to keep test isolation clean
|
||||||
|
* (tests that exercise ring timeouts can register their own stub via
|
||||||
|
* `connectionManager.setRingTimeoutFanoutHook`).
|
||||||
|
*/
|
||||||
|
export function registerCallRelayHooks(): void {
|
||||||
|
connectionManager.setRingTimeoutFanoutHook(async (dmChannelId, callerId) => {
|
||||||
|
const failures = await sendFederatedCallEnd(dmChannelId, callerId);
|
||||||
|
if (failures.length > 0) {
|
||||||
|
console.warn('[federation] Ring-timeout fan-out had failures:', failures);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// ─── Test-only exports ──────────────────────────────────────────────────────
|
// ─── Test-only exports ──────────────────────────────────────────────────────
|
||||||
/** Direct export for unit tests — do not use in production code paths. */
|
/** Direct export for unit tests — do not use in production code paths. */
|
||||||
export const handleDmCallAcceptForTest = handleDmCallAccept;
|
export const handleDmCallAcceptForTest = handleDmCallAccept;
|
||||||
|
|||||||
@@ -99,6 +99,9 @@ class ConnectionManager {
|
|||||||
private pendingOfflineTimeouts: Map<string, NodeJS.Timeout> = new Map();
|
private pendingOfflineTimeouts: Map<string, NodeJS.Timeout> = new Map();
|
||||||
// roomId → Timeout for ringing DM rooms (60s auto-cleanup)
|
// roomId → Timeout for ringing DM rooms (60s auto-cleanup)
|
||||||
private ringingTimeouts: Map<string, NodeJS.Timeout> = new Map();
|
private ringingTimeouts: Map<string, NodeJS.Timeout> = new Map();
|
||||||
|
// Callback registered by events.ts to fan dm_call_end out to peers on ring timeout.
|
||||||
|
// Null during startup — ring timeouts that fire before registration simply no-op (there are no peers to notify before boot completes).
|
||||||
|
private ringTimeoutFanoutHook: ((dmChannelId: string, callerId: string) => Promise<void>) | null = null;
|
||||||
/** Federated calls where this instance is NOT the host. Keyed by federatedId. */
|
/** Federated calls where this instance is NOT the host. Keyed by federatedId. */
|
||||||
private federatedCalls: Map<string, FederatedCallEntry> = new Map();
|
private federatedCalls: Map<string, FederatedCallEntry> = new Map();
|
||||||
private federatedCallTimeouts: Map<string, NodeJS.Timeout> = new Map();
|
private federatedCallTimeouts: Map<string, NodeJS.Timeout> = new Map();
|
||||||
@@ -401,6 +404,11 @@ class ConnectionManager {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Register a fan-out callback invoked when a ringing DM room hits its 60s timeout. */
|
||||||
|
setRingTimeoutFanoutHook(fn: (dmChannelId: string, callerId: string) => Promise<void>): void {
|
||||||
|
this.ringTimeoutFanoutHook = fn;
|
||||||
|
}
|
||||||
|
|
||||||
/** Create a DM room in ringing state with 60s auto-cleanup. */
|
/** Create a DM room in ringing state with 60s auto-cleanup. */
|
||||||
createDmRoom(dmChannelId: string, callerId: string): boolean {
|
createDmRoom(dmChannelId: string, callerId: string): boolean {
|
||||||
const created = this.createRoom(dmChannelId, 'dm', {
|
const created = this.createRoom(dmChannelId, 'dm', {
|
||||||
@@ -415,11 +423,19 @@ class ConnectionManager {
|
|||||||
this.ringingTimeouts.delete(dmChannelId);
|
this.ringingTimeouts.delete(dmChannelId);
|
||||||
const room = this.voiceRooms.get(dmChannelId);
|
const room = this.voiceRooms.get(dmChannelId);
|
||||||
if (room && room.roomType === 'dm' && (room.metadata as DmRoomMeta).state === 'ringing') {
|
if (room && room.roomType === 'dm' && (room.metadata as DmRoomMeta).state === 'ringing') {
|
||||||
|
const ringedCallerId = (room.metadata as DmRoomMeta).callerId;
|
||||||
this.destroyRoom(dmChannelId);
|
this.destroyRoom(dmChannelId);
|
||||||
this.sendToDmMembers(dmChannelId, {
|
this.sendToDmMembers(dmChannelId, {
|
||||||
type: 'dm_call_ended',
|
type: 'dm_call_ended',
|
||||||
dmChannelId,
|
dmChannelId,
|
||||||
});
|
});
|
||||||
|
// Fan dm_call_end out to remote peers so stranded Path-A/B ringees exit the ring.
|
||||||
|
// Without this, an accept-relay failure → Alice's 60s auto-clean leaves Bob's FederatedCallEntry lingering with no terminal event.
|
||||||
|
if (this.ringTimeoutFanoutHook) {
|
||||||
|
this.ringTimeoutFanoutHook(dmChannelId, ringedCallerId).catch(err =>
|
||||||
|
console.error('[ws] ring-timeout fan-out error:', err),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}, 60_000);
|
}, 60_000);
|
||||||
this.ringingTimeouts.set(dmChannelId, timeout);
|
this.ringingTimeouts.set(dmChannelId, timeout);
|
||||||
|
|||||||
Reference in New Issue
Block a user