From 727566ef3d86b3da4797f54349ca3f433a458e88 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 12 Nov 2025 08:56:40 +0000 Subject: [PATCH 1/4] feat: Restart mic stream on visibility change and pageshow Co-authored-by: dexter --- src/app/(app)/ToggleRealtime.tsx | 19 +++- src/realtime/provider.tsx | 154 ++++++++++++++++++++++++++++++- 2 files changed, 170 insertions(+), 3 deletions(-) diff --git a/src/app/(app)/ToggleRealtime.tsx b/src/app/(app)/ToggleRealtime.tsx index e4ccf05..ad05926 100644 --- a/src/app/(app)/ToggleRealtime.tsx +++ b/src/app/(app)/ToggleRealtime.tsx @@ -181,18 +181,35 @@ export default function ToggleRealtime() { useEffect(() => { cancelInitRef.current = false let cancelled = false + let visibilityCleanup: (() => void) | null = null const run = async () => { if (cancelled) return await beginSession() } - void run() + const startWhenVisible = () => { + if (cancelled) return + void run() + } + + if (typeof document !== 'undefined' && document.visibilityState !== 'visible') { + const handleVisibility = () => { + if (document.visibilityState !== 'visible') return + document.removeEventListener('visibilitychange', handleVisibility) + startWhenVisible() + } + document.addEventListener('visibilitychange', handleVisibility) + visibilityCleanup = () => document.removeEventListener('visibilitychange', handleVisibility) + } else { + startWhenVisible() + } return () => { cancelled = true cancelInitRef.current = true startedRef.current = false + visibilityCleanup?.() stop() try { sourceRef.current?.disconnect() diff --git a/src/realtime/provider.tsx b/src/realtime/provider.tsx index 3209321..40feb78 100644 --- a/src/realtime/provider.tsx +++ b/src/realtime/provider.tsx @@ -1,6 +1,14 @@ 'use client' -import { createContext, useCallback, useContext, useMemo, useRef, useState } from 'react' +import { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState +} from 'react' import { createRealtimeSession } from '@/app/actions/realtime' @@ -20,15 +28,133 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { const [dataChannel, setDataChannel] = useState(null) const peerRef = useRef(null) const localRef = useRef(null) + const trackMonitorCleanupRef = useRef<(() => void) | null>(null) + const restartingMicRef = useRef(false) + const restartAttemptsRef = useRef<{ since: number; count: number }>({ since: 0, count: 0 }) + const restartLocalStreamRef = useRef<(reason?: string) => Promise | void>() + + const clearMonitor = useCallback(() => { + trackMonitorCleanupRef.current?.() + trackMonitorCleanupRef.current = null + }, []) + + const attachTrackMonitors = useCallback((stream: MediaStream) => { + clearMonitor() + const [track] = stream.getAudioTracks() + if (!track) return + + console.log('[realtime] monitoring microphone track', { id: track.id }) + + const scheduleRestart = (reason: string) => { + console.warn('[realtime] microphone track lost', { reason }) + try { + restartLocalStreamRef.current?.(reason) + } catch (error) { + console.error('[realtime] mic restart handler threw', error) + } + } + + let muteTimer: ReturnType | null = null + + const handleEnded = () => scheduleRestart('track-ended') + const handleMute = () => { + if (!track.muted) return + if (muteTimer) clearTimeout(muteTimer) + muteTimer = setTimeout(() => { + if (!track.muted) return + scheduleRestart('track-muted') + }, 650) + } + const handleUnmute = () => { + if (!muteTimer) return + clearTimeout(muteTimer) + muteTimer = null + } + + track.addEventListener('ended', handleEnded) + track.addEventListener('mute', handleMute) + track.addEventListener('unmute', handleUnmute) + + trackMonitorCleanupRef.current = () => { + if (muteTimer) { + clearTimeout(muteTimer) + muteTimer = null + } + track.removeEventListener('ended', handleEnded) + track.removeEventListener('mute', handleMute) + track.removeEventListener('unmute', handleUnmute) + } + }, [clearMonitor]) + + const restartLocalStream = useCallback( + async (reason = 'unknown') => { + if (!peerRef.current) { + console.warn('[realtime] skipping microphone restart, no peer connection', { reason }) + return + } + if (restartingMicRef.current) { + console.log('[realtime] microphone restart already running', { reason }) + return + } + + const now = Date.now() + const windowMs = 8_000 + if (now - restartAttemptsRef.current.since > windowMs) { + restartAttemptsRef.current = { since: now, count: 0 } + } + if (restartAttemptsRef.current.count >= 3) { + console.warn('[realtime] suppressing microphone restart, too many attempts', { + reason + }) + return + } + restartAttemptsRef.current.count += 1 + restartingMicRef.current = true + + try { + console.log('[realtime] restarting microphone stream', { reason }) + const fresh = await navigator.mediaDevices.getUserMedia({ audio: true }) + attachTrackMonitors(fresh) + const previous = localRef.current + localRef.current = fresh + const [track] = fresh.getAudioTracks() + if (!track) throw new Error('Mic restart produced no audio track') + const sender = peerRef.current + .getSenders() + .find(candidate => candidate.track?.kind === 'audio') + if (sender) { + await sender.replaceTrack(track) + console.log('[realtime] replaced outbound audio track', { trackId: track.id }) + } else { + peerRef.current.addTrack(track, fresh) + console.log('[realtime] added outbound audio track after restart', { + trackId: track.id + }) + } + for (const prevTrack of previous?.getTracks() ?? []) prevTrack.stop() + restartAttemptsRef.current = { since: now, count: 0 } + } catch (error) { + console.error('[realtime] failed to restart microphone', error) + } finally { + restartingMicRef.current = false + } + }, + [attachTrackMonitors] + ) + + restartLocalStreamRef.current = restartLocalStream const cleanup = useCallback(() => { peerRef.current?.close() peerRef.current = null for (const track of localRef.current?.getTracks() ?? []) track.stop() localRef.current = null + clearMonitor() + restartingMicRef.current = false + restartAttemptsRef.current = { since: 0, count: 0 } setRemoteStream(null) setDataChannel(null) - }, []) + }, [clearMonitor]) const start = useCallback( async (opts?: { instructions?: string; voice?: string; model?: string }) => { @@ -47,6 +173,7 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { audioTracks: mic.getAudioTracks().length }) localRef.current = mic + attachTrackMonitors(mic) const pc = new RTCPeerConnection() peerRef.current = pc @@ -197,6 +324,29 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { [dataChannel, remoteStream, start, updateInstructions, updateVoice, stop] ) + useEffect(() => { + if (typeof document === 'undefined') return + const handleVisibility = () => { + if (document.visibilityState !== 'visible') return + const track = localRef.current?.getAudioTracks()[0] + if (track && track.readyState === 'live') return + void restartLocalStream('visibilitychange') + } + document.addEventListener('visibilitychange', handleVisibility) + return () => document.removeEventListener('visibilitychange', handleVisibility) + }, [restartLocalStream]) + + useEffect(() => { + if (typeof window === 'undefined') return + const handlePageShow = (event: PageTransitionEvent) => { + if (event.persisted) { + void restartLocalStream('pageshow') + } + } + window.addEventListener('pageshow', handlePageShow) + return () => window.removeEventListener('pageshow', handlePageShow) + }, [restartLocalStream]) + return {children} } From 246d776f9228ccbaf59b446bfce197a178d3c183 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 12 Nov 2025 08:58:11 +0000 Subject: [PATCH 2/4] Refactor: Allow restartLocalStreamRef to be null Co-authored-by: dexter --- src/realtime/provider.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/realtime/provider.tsx b/src/realtime/provider.tsx index 40feb78..0474c4b 100644 --- a/src/realtime/provider.tsx +++ b/src/realtime/provider.tsx @@ -31,7 +31,7 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { const trackMonitorCleanupRef = useRef<(() => void) | null>(null) const restartingMicRef = useRef(false) const restartAttemptsRef = useRef<{ since: number; count: number }>({ since: 0, count: 0 }) - const restartLocalStreamRef = useRef<(reason?: string) => Promise | void>() + const restartLocalStreamRef = useRef<((reason?: string) => Promise | void) | null>(null) const clearMonitor = useCallback(() => { trackMonitorCleanupRef.current?.() From 65554ebfb0359e0c7d8d66b87841a92118c39fb4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 12 Nov 2025 09:16:43 +0000 Subject: [PATCH 3/4] Refactor realtime connection and error handling This commit refactors the realtime connection logic to improve robustness and error handling. It introduces a new `scheduleSessionRestart` function to manage connection retries and implements more comprehensive monitoring of the local microphone track. Additionally, it updates the `start` function to handle concurrent calls and ensures proper cleanup of resources. The changes aim to make the realtime connection more stable and resilient to network issues and device changes. Co-authored-by: dexter --- src/app/(app)/ToggleRealtime.tsx | 1 + src/realtime/provider.tsx | 454 +++++++++++++++++-------------- 2 files changed, 245 insertions(+), 210 deletions(-) diff --git a/src/app/(app)/ToggleRealtime.tsx b/src/app/(app)/ToggleRealtime.tsx index ad05926..72bc6d8 100644 --- a/src/app/(app)/ToggleRealtime.tsx +++ b/src/app/(app)/ToggleRealtime.tsx @@ -112,6 +112,7 @@ export default function ToggleRealtime() { let cancelled = false const connectRemoteAudio = async () => { + if (cancelled) return console.log('[lilac] remoteStream updated', { hasStream: Boolean(remoteStream), tracks: remoteStream?.getTracks().length diff --git a/src/realtime/provider.tsx b/src/realtime/provider.tsx index 0474c4b..025e5c0 100644 --- a/src/realtime/provider.tsx +++ b/src/realtime/provider.tsx @@ -29,250 +29,275 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { const peerRef = useRef(null) const localRef = useRef(null) const trackMonitorCleanupRef = useRef<(() => void) | null>(null) - const restartingMicRef = useRef(false) - const restartAttemptsRef = useRef<{ since: number; count: number }>({ since: 0, count: 0 }) - const restartLocalStreamRef = useRef<((reason?: string) => Promise | void) | null>(null) + const lastStartOptsRef = useRef<{ instructions?: string; voice?: string; model?: string } | undefined>( + undefined + ) + const startInProgressRef = useRef(false) + const restartingSessionRef = useRef(false) + const restartTimerRef = useRef | null>(null) + const startRef = useRef(null) - const clearMonitor = useCallback(() => { + const cleanup = useCallback(() => { + peerRef.current?.close() + peerRef.current = null + for (const track of localRef.current?.getTracks() ?? []) track.stop() + localRef.current = null trackMonitorCleanupRef.current?.() trackMonitorCleanupRef.current = null - }, []) - - const attachTrackMonitors = useCallback((stream: MediaStream) => { - clearMonitor() - const [track] = stream.getAudioTracks() - if (!track) return - - console.log('[realtime] monitoring microphone track', { id: track.id }) - - const scheduleRestart = (reason: string) => { - console.warn('[realtime] microphone track lost', { reason }) - try { - restartLocalStreamRef.current?.(reason) - } catch (error) { - console.error('[realtime] mic restart handler threw', error) - } + if (restartTimerRef.current) { + clearTimeout(restartTimerRef.current) + restartTimerRef.current = null } + setRemoteStream(null) + setDataChannel(null) + }, []) - let muteTimer: ReturnType | null = null + const scheduleSessionRestart = useCallback( + (reason: string, delay = 300) => { + if (!lastStartOptsRef.current) return + if (restartTimerRef.current) clearTimeout(restartTimerRef.current) + restartTimerRef.current = setTimeout(async () => { + restartTimerRef.current = null + if (startInProgressRef.current || restartingSessionRef.current) { + scheduleSessionRestart(reason, Math.min(delay * 2, 2000)) + return + } + const startFn = startRef.current + if (!startFn) return + try { + restartingSessionRef.current = true + console.warn('[realtime] restarting session', { reason, delay }) + cleanup() + await startFn(lastStartOptsRef.current) + } catch (error) { + console.error('[realtime] session restart failed', error) + scheduleSessionRestart('retry-after-failure', Math.min(delay * 2, 2000)) + } finally { + restartingSessionRef.current = false + } + }, delay) + }, + [cleanup] + ) - const handleEnded = () => scheduleRestart('track-ended') - const handleMute = () => { - if (!track.muted) return - if (muteTimer) clearTimeout(muteTimer) - muteTimer = setTimeout(() => { - if (!track.muted) return - scheduleRestart('track-muted') - }, 650) - } - const handleUnmute = () => { - if (!muteTimer) return - clearTimeout(muteTimer) - muteTimer = null - } + const attachLocalTrackMonitor = useCallback( + (stream: MediaStream) => { + trackMonitorCleanupRef.current?.() + trackMonitorCleanupRef.current = null + const [track] = stream.getAudioTracks() + if (!track) return - track.addEventListener('ended', handleEnded) - track.addEventListener('mute', handleMute) - track.addEventListener('unmute', handleUnmute) + console.log('[realtime] monitoring local microphone track', { + id: track.id, + label: track.label + }) - trackMonitorCleanupRef.current = () => { - if (muteTimer) { - clearTimeout(muteTimer) - muteTimer = null + let mutedTimer: ReturnType | null = null + const flushMutedTimer = () => { + if (mutedTimer) { + clearTimeout(mutedTimer) + mutedTimer = null + } } - track.removeEventListener('ended', handleEnded) - track.removeEventListener('mute', handleMute) - track.removeEventListener('unmute', handleUnmute) - } - }, [clearMonitor]) - const restartLocalStream = useCallback( - async (reason = 'unknown') => { - if (!peerRef.current) { - console.warn('[realtime] skipping microphone restart, no peer connection', { reason }) - return - } - if (restartingMicRef.current) { - console.log('[realtime] microphone restart already running', { reason }) - return + const handleTrackLost = (reason: string) => { + console.warn('[realtime] microphone track inactive', { reason, id: track.id }) + flushMutedTimer() + scheduleSessionRestart(reason, reason === 'ended' ? 150 : 400) } - const now = Date.now() - const windowMs = 8_000 - if (now - restartAttemptsRef.current.since > windowMs) { - restartAttemptsRef.current = { since: now, count: 0 } - } - if (restartAttemptsRef.current.count >= 3) { - console.warn('[realtime] suppressing microphone restart, too many attempts', { - reason - }) - return + const handleEnded = () => handleTrackLost('ended') + const handleMute = () => { + if (!track.muted) return + flushMutedTimer() + mutedTimer = setTimeout(() => { + if (track.muted) handleTrackLost('mute-timeout') + }, 600) } - restartAttemptsRef.current.count += 1 - restartingMicRef.current = true + const handleUnmute = () => flushMutedTimer() - try { - console.log('[realtime] restarting microphone stream', { reason }) - const fresh = await navigator.mediaDevices.getUserMedia({ audio: true }) - attachTrackMonitors(fresh) - const previous = localRef.current - localRef.current = fresh - const [track] = fresh.getAudioTracks() - if (!track) throw new Error('Mic restart produced no audio track') - const sender = peerRef.current - .getSenders() - .find(candidate => candidate.track?.kind === 'audio') - if (sender) { - await sender.replaceTrack(track) - console.log('[realtime] replaced outbound audio track', { trackId: track.id }) - } else { - peerRef.current.addTrack(track, fresh) - console.log('[realtime] added outbound audio track after restart', { - trackId: track.id - }) + track.addEventListener('ended', handleEnded) + track.addEventListener('mute', handleMute) + track.addEventListener('unmute', handleUnmute) + + const healthInterval = setInterval(() => { + if (track.readyState !== 'live') { + handleTrackLost(`state-${track.readyState}`) } - for (const prevTrack of previous?.getTracks() ?? []) prevTrack.stop() - restartAttemptsRef.current = { since: now, count: 0 } - } catch (error) { - console.error('[realtime] failed to restart microphone', error) - } finally { - restartingMicRef.current = false + }, 1200) + + trackMonitorCleanupRef.current = () => { + flushMutedTimer() + clearInterval(healthInterval) + track.removeEventListener('ended', handleEnded) + track.removeEventListener('mute', handleMute) + track.removeEventListener('unmute', handleUnmute) } }, - [attachTrackMonitors] + [scheduleSessionRestart] ) - restartLocalStreamRef.current = restartLocalStream - - const cleanup = useCallback(() => { - peerRef.current?.close() - peerRef.current = null - for (const track of localRef.current?.getTracks() ?? []) track.stop() - localRef.current = null - clearMonitor() - restartingMicRef.current = false - restartAttemptsRef.current = { since: 0, count: 0 } - setRemoteStream(null) - setDataChannel(null) - }, [clearMonitor]) - const start = useCallback( async (opts?: { instructions?: string; voice?: string; model?: string }) => { + if (startInProgressRef.current) { + console.log('[realtime] start() ignored, already in progress') + return + } + startInProgressRef.current = true console.log('[realtime] start() called', { opts }) - const session = await createRealtimeSession({ - instructions: opts?.instructions, - model: opts?.model, - voice: opts?.voice - }) - const clientSecret = session.client_secret.value - const model = session.model - console.log('[realtime] created session', { expires_at: session.expires_at, model }) + lastStartOptsRef.current = opts + try { + const session = await createRealtimeSession({ + instructions: opts?.instructions, + model: opts?.model, + voice: opts?.voice + }) + const clientSecret = session.client_secret.value + const model = session.model + console.log('[realtime] created session', { expires_at: session.expires_at, model }) - const mic = await navigator.mediaDevices.getUserMedia({ audio: true }) - console.log('[realtime] obtained microphone stream', { - audioTracks: mic.getAudioTracks().length - }) - localRef.current = mic - attachTrackMonitors(mic) + const mic = await navigator.mediaDevices.getUserMedia({ audio: true }) + console.log('[realtime] obtained microphone stream', { + audioTracks: mic.getAudioTracks().length + }) + localRef.current = mic + attachLocalTrackMonitor(mic) - const pc = new RTCPeerConnection() - peerRef.current = pc - pc.addEventListener('icecandidate', e => { - console.log('[realtime] icecandidate', { candidate: Boolean(e.candidate) }) - }) - pc.addEventListener('icegatheringstatechange', () => { - console.log('[realtime] icegatheringstatechange', pc.iceGatheringState) - }) - pc.addEventListener('signalingstatechange', () => { - console.log('[realtime] signalingstatechange', pc.signalingState) - }) - pc.addEventListener('connectionstatechange', () => { - console.log('[realtime] connectionstatechange', pc.connectionState) - }) - // Guard optional event for browsers that support it - ;( - pc as unknown as { addEventListener?: (t: string, cb: (e: unknown) => void) => void } - ).addEventListener?.('icecandidateerror', (e: unknown) => { - console.warn('[realtime] icecandidateerror', e) - }) + const pc = new RTCPeerConnection() + peerRef.current = pc + pc.addEventListener('icecandidate', e => { + console.log('[realtime] icecandidate', { candidate: Boolean(e.candidate) }) + }) + pc.addEventListener('icegatheringstatechange', () => { + console.log('[realtime] icegatheringstatechange', pc.iceGatheringState) + }) + pc.addEventListener('signalingstatechange', () => { + console.log('[realtime] signalingstatechange', pc.signalingState) + }) + pc.addEventListener('connectionstatechange', () => { + console.log('[realtime] connectionstatechange', pc.connectionState) + if (pc.connectionState === 'failed' || pc.connectionState === 'closed') { + scheduleSessionRestart(`connection-${pc.connectionState}`, 400) + } else if (pc.connectionState === 'disconnected') { + scheduleSessionRestart('connection-disconnected', 900) + } + }) + // Guard optional event for browsers that support it + ;( + pc as unknown as { addEventListener?: (t: string, cb: (e: unknown) => void) => void } + ).addEventListener?.('icecandidateerror', (e: unknown) => { + console.warn('[realtime] icecandidateerror', e) + }) - pc.addEventListener('track', event => { - const [first] = event.streams - if (!first) return - setRemoteStream(first) - console.log('[realtime] remote track added', { - kind: event.track.kind, - remoteAudioTracks: first.getAudioTracks().length + pc.addEventListener('track', event => { + const [first] = event.streams + if (!first) return + setRemoteStream(first) + console.log('[realtime] remote track added', { + kind: event.track.kind, + remoteAudioTracks: first.getAudioTracks().length + }) + first.addEventListener( + 'removetrack', + () => scheduleSessionRestart('remote-track-removed', 600), + { once: true } + ) }) - }) - for (const track of mic.getTracks()) { - pc.addTrack(track, mic) - } + for (const track of mic.getTracks()) { + pc.addTrack(track, mic) + } - const dc = pc.createDataChannel('oai-events') - setDataChannel(dc) - dc.addEventListener('open', () => { - console.log('[realtime] datachannel open') - try { - if (opts?.voice || opts?.instructions) { + const dc = pc.createDataChannel('oai-events') + setDataChannel(dc) + dc.addEventListener('open', () => { + console.log('[realtime] datachannel open') + // Ensure session settings (voice/instructions) are applied then trigger a first response. + try { + if (opts?.voice || opts?.instructions) { + dc.send( + JSON.stringify({ + session: { + ...(opts?.voice ? { voice: opts.voice } : {}), + ...(opts?.instructions ? { instructions: opts.instructions } : {}) + }, + type: 'session.update' + }) + ) + console.log('[realtime] sent session.update') + } + } catch {} + try { dc.send( JSON.stringify({ - session: { - ...(opts?.voice ? { voice: opts.voice } : {}), - ...(opts?.instructions ? { instructions: opts.instructions } : {}) + response: { + // If instructions provided, model may greet appropriately; otherwise send a short greeting. + ...(opts?.instructions ? {} : { instructions: 'Hello! I am ready to translate.' }) }, - type: 'session.update' + type: 'response.create' }) ) - console.log('[realtime] sent session.update') + console.log('[realtime] sent response.create') + } catch {} + }) + dc.addEventListener('close', () => { + console.log('[realtime] datachannel close') + }) + dc.addEventListener('message', e => { + try { + const msg = JSON.parse(String(e.data)) + console.log('[realtime] dc message', msg?.type ?? 'unknown', msg) + } catch { + console.log('[realtime] dc message (text)', String(e.data)) } - } catch {} - }) - dc.addEventListener('close', () => { - console.log('[realtime] datachannel close') - }) - dc.addEventListener('message', e => { - try { - const msg = JSON.parse(String(e.data)) - console.log('[realtime] dc message', msg?.type ?? 'unknown', msg) - } catch { - console.log('[realtime] dc message (text)', String(e.data)) - } - }) + }) - const offer = await pc.createOffer() - console.log('[realtime] created local offer', { sdpBytes: offer.sdp?.length ?? 0 }) - await pc.setLocalDescription(offer) - console.log('[realtime] setLocalDescription') - - const resp = await fetch( - `https://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`, - { - body: offer.sdp ?? '', - headers: { - Authorization: `Bearer ${clientSecret}`, - 'Content-Type': 'application/sdp', - 'OpenAI-Beta': 'realtime=v1' - }, - method: 'POST' + const offer = await pc.createOffer() + console.log('[realtime] created local offer', { sdpBytes: offer.sdp?.length ?? 0 }) + await pc.setLocalDescription(offer) + console.log('[realtime] setLocalDescription') + + const resp = await fetch( + `https://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`, + { + body: offer.sdp ?? '', + headers: { + Authorization: `Bearer ${clientSecret}`, + 'Content-Type': 'application/sdp', + 'OpenAI-Beta': 'realtime=v1' + }, + method: 'POST' + } + ) + console.log('[realtime] handshake response', { status: resp.status }) + if (!resp.ok) { + const body = await resp.text().catch(() => '') + console.error('[realtime] handshake failed', resp.status, body) + throw new Error(`Realtime handshake failed (${resp.status})`) } - ) - console.log('[realtime] handshake response', { status: resp.status }) - if (!resp.ok) { - const body = await resp.text().catch(() => '') - console.error('[realtime] handshake failed', resp.status, body) - throw new Error(`Realtime handshake failed (${resp.status})`) - } - const answer = await resp.text() - console.log('[realtime] received remote answer', { sdpBytes: answer.length }) - await pc.setRemoteDescription({ sdp: answer, type: 'answer' }) - console.log('[realtime] setRemoteDescription complete') + const answer = await resp.text() + console.log('[realtime] received remote answer', { sdpBytes: answer.length }) + await pc.setRemoteDescription({ sdp: answer, type: 'answer' }) + console.log('[realtime] setRemoteDescription complete') + } catch (error) { + console.error('[realtime] start() failed', error) + cleanup() + throw error + } finally { + startInProgressRef.current = false + } }, - [] + [attachLocalTrackMonitor, cleanup, scheduleSessionRestart] ) + useEffect(() => { + startRef.current = start + return () => { + startRef.current = null + } + }, [start]) + const stop = useCallback(() => { console.log('[realtime] stop() called') cleanup() @@ -329,23 +354,32 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { const handleVisibility = () => { if (document.visibilityState !== 'visible') return const track = localRef.current?.getAudioTracks()[0] - if (track && track.readyState === 'live') return - void restartLocalStream('visibilitychange') + if (track && track.readyState === 'live' && !track.muted) return + scheduleSessionRestart('visibility-resume', 250) } document.addEventListener('visibilitychange', handleVisibility) return () => document.removeEventListener('visibilitychange', handleVisibility) - }, [restartLocalStream]) + }, [scheduleSessionRestart]) useEffect(() => { if (typeof window === 'undefined') return const handlePageShow = (event: PageTransitionEvent) => { if (event.persisted) { - void restartLocalStream('pageshow') + scheduleSessionRestart('pageshow-persisted', 250) } } window.addEventListener('pageshow', handlePageShow) - return () => window.removeEventListener('pageshow', handlePageShow) - }, [restartLocalStream]) + const handleFocus = () => { + const track = localRef.current?.getAudioTracks()[0] + if (track && track.readyState === 'live') return + scheduleSessionRestart('window-focus', 400) + } + window.addEventListener('focus', handleFocus) + return () => { + window.removeEventListener('pageshow', handlePageShow) + window.removeEventListener('focus', handleFocus) + } + }, [scheduleSessionRestart]) return {children} } From 39beb8f0d795dbebcf82f93814d5c01b2b76001e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 12 Nov 2025 09:30:23 +0000 Subject: [PATCH 4/4] Refactor realtime session management and UI This commit restructures the realtime session logic, improving state management and error handling. It also updates the UI to provide a clearer user experience for starting and stopping the listening session. Co-authored-by: dexter --- src/app/(app)/ToggleRealtime.tsx | 259 +++++++++++--------- src/realtime/provider.tsx | 406 ++++++++++--------------------- 2 files changed, 279 insertions(+), 386 deletions(-) diff --git a/src/app/(app)/ToggleRealtime.tsx b/src/app/(app)/ToggleRealtime.tsx index 72bc6d8..5932274 100644 --- a/src/app/(app)/ToggleRealtime.tsx +++ b/src/app/(app)/ToggleRealtime.tsx @@ -37,17 +37,25 @@ const languagePhrases = [ { code: 'it', text: 'Presentati' } ] -type ConnectionState = 'idle' | 'requesting' | 'ready' | 'error' +type SessionState = 'idle' | 'requesting' | 'listening' | 'error' export default function ToggleRealtime() { const { start, stop, remoteStream } = useRealtimeVoiceSession() - const [connectionState, setConnectionState] = useState('idle') + const [sessionState, setSessionState] = useState('idle') const [errorMessage, setErrorMessage] = useState(null) const [languageOrder, setLanguageOrder] = useState(languagePhrases) + const [isStandalone, setIsStandalone] = useState(false) const audioContextRef = useRef(null) const sourceRef = useRef(null) - const startedRef = useRef(false) - const cancelInitRef = useRef(false) + const startPendingRef = useRef(false) + + useEffect(() => { + if (typeof window === 'undefined') return + const standalone = + window.matchMedia?.('(display-mode: standalone)').matches || + (window.navigator as unknown as { standalone?: boolean }).standalone === true + setIsStandalone(Boolean(standalone)) + }, []) useEffect(() => { if (typeof navigator === 'undefined') return @@ -79,11 +87,11 @@ export default function ToggleRealtime() { setLanguageOrder(prioritized) }, []) - const ensureAudioContext = useCallback(() => { + const ensureAudioContext = useCallback(async () => { const Ctx = window.AudioContext ?? (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext - if (!Ctx) throw new Error('AudioContext is not supported in this browser') + if (!Ctx) throw new Error('AudioContext is not supported on this device') if (!audioContextRef.current) { audioContextRef.current = new Ctx() @@ -91,145 +99,136 @@ export default function ToggleRealtime() { } if (audioContextRef.current.state === 'suspended') { - void audioContextRef.current - .resume() - .then(() => { - console.log('[lilac] AudioContext resumed', { - state: audioContextRef.current?.state - }) - }) - .catch(error => { - console.warn('[lilac] failed to resume AudioContext', error) - }) - } - - return audioContextRef.current - }, []) - - useEffect(() => { - if (!remoteStream) return - - let cancelled = false - - const connectRemoteAudio = async () => { - if (cancelled) return - console.log('[lilac] remoteStream updated', { - hasStream: Boolean(remoteStream), - tracks: remoteStream?.getTracks().length - }) - - if (!remoteStream.getAudioTracks().length) { - const onAddTrack = () => { - remoteStream.removeEventListener('addtrack', onAddTrack as EventListener) - void connectRemoteAudio() - } - remoteStream.addEventListener('addtrack', onAddTrack as EventListener) - return - } - try { - const ctx = ensureAudioContext() - if (!ctx || cancelled) return - const src = ctx.createMediaStreamSource(remoteStream) - sourceRef.current = src - src.connect(ctx.destination) + await audioContextRef.current.resume() + console.log('[lilac] AudioContext resumed', { + state: audioContextRef.current?.state + }) } catch (error) { - console.error('[lilac] failed to connect remote audio', error) + console.warn('[lilac] failed to resume AudioContext', error) } } - void connectRemoteAudio() + return audioContextRef.current + }, []) - return () => { - cancelled = true - console.log('[lilac] cleaning audio graph') - try { - sourceRef.current?.disconnect() - } catch {} - sourceRef.current = null + const cleanupAudioGraph = useCallback(() => { + try { + sourceRef.current?.disconnect() + } catch {} + sourceRef.current = null + if (audioContextRef.current) { try { - void audioContextRef.current?.close() + void audioContextRef.current.close() } catch {} audioContextRef.current = null } - }, [remoteStream, ensureAudioContext]) + }, []) - const beginSession = useCallback(async () => { - if (startedRef.current) return - startedRef.current = true - setConnectionState('requesting') + const handleStop = useCallback(() => { + startPendingRef.current = false + stop() + cleanupAudioGraph() + setSessionState('idle') setErrorMessage(null) + }, [cleanupAudioGraph, stop]) + const handleStart = useCallback(async () => { + if (startPendingRef.current) return + startPendingRef.current = true + setErrorMessage(null) + setSessionState('requesting') try { - ensureAudioContext() + await ensureAudioContext() await start({ instructions: defaultPrompt, voice: 'verse' }) - if (cancelInitRef.current) { - startedRef.current = false - return - } - setConnectionState('ready') + setSessionState('listening') } catch (error) { - console.error('[lilac] failed to start realtime session', error) - startedRef.current = false - if (cancelInitRef.current) return - setConnectionState('error') + console.error('[lilac] unable to start session', error) const message = - error instanceof Error ? error.message : 'Something went wrong while starting Lilac.' + error instanceof Error + ? error.message + : 'Something went wrong while requesting the microphone.' setErrorMessage(message) + setSessionState('error') + cleanupAudioGraph() + stop() + } finally { + startPendingRef.current = false } - }, [ensureAudioContext, start]) + }, [cleanupAudioGraph, ensureAudioContext, start, stop]) useEffect(() => { - cancelInitRef.current = false + if (!remoteStream) return cleanupAudioGraph + let cancelled = false - let visibilityCleanup: (() => void) | null = null - const run = async () => { - if (cancelled) return - await beginSession() + const connect = async () => { + try { + const ctx = await ensureAudioContext() + if (!ctx || cancelled) return + if (!remoteStream.getAudioTracks().length) { + const handleAddTrack = () => { + remoteStream.removeEventListener('addtrack', handleAddTrack as EventListener) + void connect() + } + remoteStream.addEventListener('addtrack', handleAddTrack as EventListener) + return + } + const node = ctx.createMediaStreamSource(remoteStream) + try { + sourceRef.current?.disconnect() + } catch {} + sourceRef.current = node + node.connect(ctx.destination) + } catch (error) { + if (!cancelled) { + console.error('[lilac] failed to wire remote audio', error) + } + } } - const startWhenVisible = () => { - if (cancelled) return - void run() + void connect() + + return () => { + cancelled = true + cleanupAudioGraph() } + }, [cleanupAudioGraph, ensureAudioContext, remoteStream]) - if (typeof document !== 'undefined' && document.visibilityState !== 'visible') { - const handleVisibility = () => { - if (document.visibilityState !== 'visible') return - document.removeEventListener('visibilitychange', handleVisibility) - startWhenVisible() + useEffect(() => { + if (typeof document === 'undefined') return + const handleVisibility = () => { + if (document.visibilityState === 'hidden' && sessionState !== 'idle') { + console.log('[lilac] document hidden -> stopping session') + handleStop() } - document.addEventListener('visibilitychange', handleVisibility) - visibilityCleanup = () => document.removeEventListener('visibilitychange', handleVisibility) - } else { - startWhenVisible() } + document.addEventListener('visibilitychange', handleVisibility) + return () => document.removeEventListener('visibilitychange', handleVisibility) + }, [handleStop, sessionState]) + useEffect(() => { + if (typeof window === 'undefined') return + const handleBlur = () => { + if (sessionState !== 'idle') { + console.log('[lilac] window blur -> stopping session') + handleStop() + } + } + window.addEventListener('beforeunload', handleStop) + window.addEventListener('blur', handleBlur) return () => { - cancelled = true - cancelInitRef.current = true - startedRef.current = false - visibilityCleanup?.() - stop() - try { - sourceRef.current?.disconnect() - } catch {} - sourceRef.current = null - try { - void audioContextRef.current?.close() - } catch {} - audioContextRef.current = null + window.removeEventListener('beforeunload', handleStop) + window.removeEventListener('blur', handleBlur) } - }, [beginSession, stop]) + }, [handleStop, sessionState]) const statusText = useMemo(() => { - if (connectionState === 'requesting') return 'Requesting microphone…' - if (connectionState === 'ready') return 'Listening' - if (connectionState === 'error') - return errorMessage ?? 'Unable to start. Check microphone permissions.' - return 'Preparing Lilac…' - }, [connectionState, errorMessage]) + if (sessionState === 'requesting') return 'Requesting microphone…' + if (sessionState === 'listening') return 'Listening' + if (sessionState === 'error') return errorMessage ?? 'Unable to start. Check microphone permissions.' + return 'Tap start to begin listening.' + }, [errorMessage, sessionState]) const [activeIndex, setActiveIndex] = useState(0) @@ -251,12 +250,13 @@ export default function ToggleRealtime() {
Lilac + {isStandalone && Home Screen}
-
+
+
+ + +
{statusText} + {sessionState === 'error' && errorMessage ? ( + + {errorMessage} + + ) : null} + {sessionState === 'idle' && ( + + {isStandalone + ? 'If the mic stops after reopening, tap Start again to refresh permissions.' + : 'For a full-screen experience add Lilac to your home screen.'} + + )}
) diff --git a/src/realtime/provider.tsx b/src/realtime/provider.tsx index 025e5c0..aaa2c9e 100644 --- a/src/realtime/provider.tsx +++ b/src/realtime/provider.tsx @@ -28,276 +28,145 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { const [dataChannel, setDataChannel] = useState(null) const peerRef = useRef(null) const localRef = useRef(null) - const trackMonitorCleanupRef = useRef<(() => void) | null>(null) - const lastStartOptsRef = useRef<{ instructions?: string; voice?: string; model?: string } | undefined>( - undefined - ) - const startInProgressRef = useRef(false) - const restartingSessionRef = useRef(false) - const restartTimerRef = useRef | null>(null) - const startRef = useRef(null) const cleanup = useCallback(() => { peerRef.current?.close() peerRef.current = null for (const track of localRef.current?.getTracks() ?? []) track.stop() localRef.current = null - trackMonitorCleanupRef.current?.() - trackMonitorCleanupRef.current = null - if (restartTimerRef.current) { - clearTimeout(restartTimerRef.current) - restartTimerRef.current = null - } setRemoteStream(null) setDataChannel(null) }, []) - const scheduleSessionRestart = useCallback( - (reason: string, delay = 300) => { - if (!lastStartOptsRef.current) return - if (restartTimerRef.current) clearTimeout(restartTimerRef.current) - restartTimerRef.current = setTimeout(async () => { - restartTimerRef.current = null - if (startInProgressRef.current || restartingSessionRef.current) { - scheduleSessionRestart(reason, Math.min(delay * 2, 2000)) - return - } - const startFn = startRef.current - if (!startFn) return - try { - restartingSessionRef.current = true - console.warn('[realtime] restarting session', { reason, delay }) - cleanup() - await startFn(lastStartOptsRef.current) - } catch (error) { - console.error('[realtime] session restart failed', error) - scheduleSessionRestart('retry-after-failure', Math.min(delay * 2, 2000)) - } finally { - restartingSessionRef.current = false - } - }, delay) - }, - [cleanup] - ) - - const attachLocalTrackMonitor = useCallback( - (stream: MediaStream) => { - trackMonitorCleanupRef.current?.() - trackMonitorCleanupRef.current = null - const [track] = stream.getAudioTracks() - if (!track) return - - console.log('[realtime] monitoring local microphone track', { - id: track.id, - label: track.label - }) - - let mutedTimer: ReturnType | null = null - const flushMutedTimer = () => { - if (mutedTimer) { - clearTimeout(mutedTimer) - mutedTimer = null - } - } - - const handleTrackLost = (reason: string) => { - console.warn('[realtime] microphone track inactive', { reason, id: track.id }) - flushMutedTimer() - scheduleSessionRestart(reason, reason === 'ended' ? 150 : 400) - } - - const handleEnded = () => handleTrackLost('ended') - const handleMute = () => { - if (!track.muted) return - flushMutedTimer() - mutedTimer = setTimeout(() => { - if (track.muted) handleTrackLost('mute-timeout') - }, 600) - } - const handleUnmute = () => flushMutedTimer() - - track.addEventListener('ended', handleEnded) - track.addEventListener('mute', handleMute) - track.addEventListener('unmute', handleUnmute) - - const healthInterval = setInterval(() => { - if (track.readyState !== 'live') { - handleTrackLost(`state-${track.readyState}`) - } - }, 1200) - - trackMonitorCleanupRef.current = () => { - flushMutedTimer() - clearInterval(healthInterval) - track.removeEventListener('ended', handleEnded) - track.removeEventListener('mute', handleMute) - track.removeEventListener('unmute', handleUnmute) - } - }, - [scheduleSessionRestart] - ) - const start = useCallback( async (opts?: { instructions?: string; voice?: string; model?: string }) => { - if (startInProgressRef.current) { - console.log('[realtime] start() ignored, already in progress') - return - } - startInProgressRef.current = true console.log('[realtime] start() called', { opts }) - lastStartOptsRef.current = opts - try { - const session = await createRealtimeSession({ - instructions: opts?.instructions, - model: opts?.model, - voice: opts?.voice - }) - const clientSecret = session.client_secret.value - const model = session.model - console.log('[realtime] created session', { expires_at: session.expires_at, model }) + const session = await createRealtimeSession({ + instructions: opts?.instructions, + model: opts?.model, + voice: opts?.voice + }) + const clientSecret = session.client_secret.value + const model = session.model + console.log('[realtime] created session', { expires_at: session.expires_at, model }) - const mic = await navigator.mediaDevices.getUserMedia({ audio: true }) - console.log('[realtime] obtained microphone stream', { - audioTracks: mic.getAudioTracks().length - }) - localRef.current = mic - attachLocalTrackMonitor(mic) + const mic = await navigator.mediaDevices.getUserMedia({ audio: true }) + console.log('[realtime] obtained microphone stream', { + audioTracks: mic.getAudioTracks().length + }) + localRef.current = mic - const pc = new RTCPeerConnection() - peerRef.current = pc - pc.addEventListener('icecandidate', e => { - console.log('[realtime] icecandidate', { candidate: Boolean(e.candidate) }) - }) - pc.addEventListener('icegatheringstatechange', () => { - console.log('[realtime] icegatheringstatechange', pc.iceGatheringState) - }) - pc.addEventListener('signalingstatechange', () => { - console.log('[realtime] signalingstatechange', pc.signalingState) - }) - pc.addEventListener('connectionstatechange', () => { - console.log('[realtime] connectionstatechange', pc.connectionState) - if (pc.connectionState === 'failed' || pc.connectionState === 'closed') { - scheduleSessionRestart(`connection-${pc.connectionState}`, 400) - } else if (pc.connectionState === 'disconnected') { - scheduleSessionRestart('connection-disconnected', 900) - } - }) - // Guard optional event for browsers that support it - ;( - pc as unknown as { addEventListener?: (t: string, cb: (e: unknown) => void) => void } - ).addEventListener?.('icecandidateerror', (e: unknown) => { - console.warn('[realtime] icecandidateerror', e) - }) + const pc = new RTCPeerConnection() + peerRef.current = pc + pc.addEventListener('icecandidate', e => { + console.log('[realtime] icecandidate', { candidate: Boolean(e.candidate) }) + }) + pc.addEventListener('icegatheringstatechange', () => { + console.log('[realtime] icegatheringstatechange', pc.iceGatheringState) + }) + pc.addEventListener('signalingstatechange', () => { + console.log('[realtime] signalingstatechange', pc.signalingState) + }) + pc.addEventListener('connectionstatechange', () => { + console.log('[realtime] connectionstatechange', pc.connectionState) + }) + // Guard optional event for browsers that support it + ;( + pc as unknown as { addEventListener?: (t: string, cb: (e: unknown) => void) => void } + ).addEventListener?.('icecandidateerror', (e: unknown) => { + console.warn('[realtime] icecandidateerror', e) + }) - pc.addEventListener('track', event => { - const [first] = event.streams - if (!first) return - setRemoteStream(first) - console.log('[realtime] remote track added', { - kind: event.track.kind, - remoteAudioTracks: first.getAudioTracks().length - }) - first.addEventListener( - 'removetrack', - () => scheduleSessionRestart('remote-track-removed', 600), - { once: true } - ) + pc.addEventListener('track', event => { + const [first] = event.streams + if (!first) return + setRemoteStream(first) + console.log('[realtime] remote track added', { + kind: event.track.kind, + remoteAudioTracks: first.getAudioTracks().length }) + }) - for (const track of mic.getTracks()) { - pc.addTrack(track, mic) - } + for (const track of mic.getTracks()) { + pc.addTrack(track, mic) + } - const dc = pc.createDataChannel('oai-events') - setDataChannel(dc) - dc.addEventListener('open', () => { - console.log('[realtime] datachannel open') - // Ensure session settings (voice/instructions) are applied then trigger a first response. - try { - if (opts?.voice || opts?.instructions) { - dc.send( - JSON.stringify({ - session: { - ...(opts?.voice ? { voice: opts.voice } : {}), - ...(opts?.instructions ? { instructions: opts.instructions } : {}) - }, - type: 'session.update' - }) - ) - console.log('[realtime] sent session.update') - } - } catch {} - try { + const dc = pc.createDataChannel('oai-events') + setDataChannel(dc) + dc.addEventListener('open', () => { + console.log('[realtime] datachannel open') + // Ensure session settings (voice/instructions) are applied then trigger a first response. + try { + if (opts?.voice || opts?.instructions) { dc.send( JSON.stringify({ - response: { - // If instructions provided, model may greet appropriately; otherwise send a short greeting. - ...(opts?.instructions ? {} : { instructions: 'Hello! I am ready to translate.' }) + session: { + ...(opts?.voice ? { voice: opts.voice } : {}), + ...(opts?.instructions ? { instructions: opts.instructions } : {}) }, - type: 'response.create' + type: 'session.update' }) ) - console.log('[realtime] sent response.create') - } catch {} - }) - dc.addEventListener('close', () => { - console.log('[realtime] datachannel close') - }) - dc.addEventListener('message', e => { - try { - const msg = JSON.parse(String(e.data)) - console.log('[realtime] dc message', msg?.type ?? 'unknown', msg) - } catch { - console.log('[realtime] dc message (text)', String(e.data)) - } - }) - - const offer = await pc.createOffer() - console.log('[realtime] created local offer', { sdpBytes: offer.sdp?.length ?? 0 }) - await pc.setLocalDescription(offer) - console.log('[realtime] setLocalDescription') - - const resp = await fetch( - `https://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`, - { - body: offer.sdp ?? '', - headers: { - Authorization: `Bearer ${clientSecret}`, - 'Content-Type': 'application/sdp', - 'OpenAI-Beta': 'realtime=v1' - }, - method: 'POST' + console.log('[realtime] sent session.update') } - ) - console.log('[realtime] handshake response', { status: resp.status }) - if (!resp.ok) { - const body = await resp.text().catch(() => '') - console.error('[realtime] handshake failed', resp.status, body) - throw new Error(`Realtime handshake failed (${resp.status})`) + } catch {} + try { + dc.send( + JSON.stringify({ + response: { + // If instructions provided, model may greet appropriately; otherwise send a short greeting. + ...(opts?.instructions ? {} : { instructions: 'Hello! I am ready to translate.' }) + }, + type: 'response.create' + }) + ) + console.log('[realtime] sent response.create') + } catch {} + }) + dc.addEventListener('close', () => { + console.log('[realtime] datachannel close') + }) + dc.addEventListener('message', e => { + try { + const msg = JSON.parse(String(e.data)) + console.log('[realtime] dc message', msg?.type ?? 'unknown', msg) + } catch { + console.log('[realtime] dc message (text)', String(e.data)) } + }) - const answer = await resp.text() - console.log('[realtime] received remote answer', { sdpBytes: answer.length }) - await pc.setRemoteDescription({ sdp: answer, type: 'answer' }) - console.log('[realtime] setRemoteDescription complete') - } catch (error) { - console.error('[realtime] start() failed', error) - cleanup() - throw error - } finally { - startInProgressRef.current = false + const offer = await pc.createOffer() + console.log('[realtime] created local offer', { sdpBytes: offer.sdp?.length ?? 0 }) + await pc.setLocalDescription(offer) + console.log('[realtime] setLocalDescription') + + const resp = await fetch( + `https://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`, + { + body: offer.sdp ?? '', + headers: { + Authorization: `Bearer ${clientSecret}`, + 'Content-Type': 'application/sdp', + 'OpenAI-Beta': 'realtime=v1' + }, + method: 'POST' + } + ) + console.log('[realtime] handshake response', { status: resp.status }) + if (!resp.ok) { + const body = await resp.text().catch(() => '') + console.error('[realtime] handshake failed', resp.status, body) + throw new Error(`Realtime handshake failed (${resp.status})`) } + + const answer = await resp.text() + console.log('[realtime] received remote answer', { sdpBytes: answer.length }) + await pc.setRemoteDescription({ sdp: answer, type: 'answer' }) + console.log('[realtime] setRemoteDescription complete') }, - [attachLocalTrackMonitor, cleanup, scheduleSessionRestart] + [] ) - useEffect(() => { - startRef.current = start - return () => { - startRef.current = null - } - }, [start]) - const stop = useCallback(() => { console.log('[realtime] stop() called') cleanup() @@ -337,6 +206,33 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { [dataChannel] ) + useEffect(() => { + if (typeof document === 'undefined') return + let hideTimer: ReturnType | null = null + + const handleVisibility = () => { + if (document.visibilityState !== 'hidden') return + if (hideTimer) clearTimeout(hideTimer) + hideTimer = setTimeout(() => { + console.log('[realtime] visibility hidden -> cleanup') + cleanup() + }, 150) + } + + const handlePageHide = () => { + console.log('[realtime] pagehide -> cleanup') + cleanup() + } + + document.addEventListener('visibilitychange', handleVisibility) + window.addEventListener('pagehide', handlePageHide) + return () => { + if (hideTimer) clearTimeout(hideTimer) + document.removeEventListener('visibilitychange', handleVisibility) + window.removeEventListener('pagehide', handlePageHide) + } + }, [cleanup]) + const value = useMemo( () => ({ dataChannel, @@ -349,38 +245,6 @@ export function RealtimeProvider({ children }: { children: React.ReactNode }) { [dataChannel, remoteStream, start, updateInstructions, updateVoice, stop] ) - useEffect(() => { - if (typeof document === 'undefined') return - const handleVisibility = () => { - if (document.visibilityState !== 'visible') return - const track = localRef.current?.getAudioTracks()[0] - if (track && track.readyState === 'live' && !track.muted) return - scheduleSessionRestart('visibility-resume', 250) - } - document.addEventListener('visibilitychange', handleVisibility) - return () => document.removeEventListener('visibilitychange', handleVisibility) - }, [scheduleSessionRestart]) - - useEffect(() => { - if (typeof window === 'undefined') return - const handlePageShow = (event: PageTransitionEvent) => { - if (event.persisted) { - scheduleSessionRestart('pageshow-persisted', 250) - } - } - window.addEventListener('pageshow', handlePageShow) - const handleFocus = () => { - const track = localRef.current?.getAudioTracks()[0] - if (track && track.readyState === 'live') return - scheduleSessionRestart('window-focus', 400) - } - window.addEventListener('focus', handleFocus) - return () => { - window.removeEventListener('pageshow', handlePageShow) - window.removeEventListener('focus', handleFocus) - } - }, [scheduleSessionRestart]) - return {children} }