From e6a48c0423baf9df4043d173c047613dcd6691fd Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Mon, 8 Dec 2025 12:53:44 +0100 Subject: [PATCH 1/9] implement session resumption --- .../backend/src/agent/gemini/session.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 1923df6..d14ea44 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -50,7 +50,7 @@ export class GeminiSession implements VoiceAgentSession { ); } - async open() { + async open( resumeHandle?: {} ) { const params: LiveConnectParameters = { model: GEMINI_MODEL, config: { @@ -74,6 +74,7 @@ export class GeminiSession implements VoiceAgentSession { proactivity: { proactiveAudio: true, }, + sessionResumption: resumeHandle }, callbacks: { onmessage: (message) => this.onMessage(message), @@ -139,6 +140,23 @@ export class GeminiSession implements VoiceAgentSession { this.handleInterrupt(); } + if (message.sessionResumptionUpdate?.newHandle) { + console.log('Gemini session resumption handle updated: %s, starting new session', message.sessionResumptionUpdate.newHandle); + const newHandle = message.sessionResumptionUpdate.newHandle; + + try { + this.close(true); + } catch (e) { + console.error('Error closing Gemini session for resumption: %o', e); + } + try { + this.open({handle: newHandle}); + } + catch (e) { + console.error('Error opening Gemini session for resumption: %o', e); + } + } + message.toolCall?.functionCalls?.forEach((call) => { switch (call.name) { case 'endGame': From f4f0222162458cf8c02da582e7206049ae8410c4 Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Mon, 8 Dec 2025 16:38:14 +0100 Subject: [PATCH 2/9] Reconnect session forever --- .../backend/src/agent/gemini/session.ts | 95 +++++++++++++------ 1 file changed, 66 insertions(+), 29 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index d14ea44..885ae31 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -14,10 +14,13 @@ export class GeminiSession implements VoiceAgentSession { private onInterrupt: (() => void) | null = null; private onAgentAudio: ((audio: Buffer) => void) | null = null; private session: Session | null = null; - private closing = false; private transcriptionParts: string[] = []; private genai: GoogleGenAI; private config: AgentConfig; + private previousHandle: string | undefined = undefined; + private closing = false; + private opening = false; + private reconnecting = false; private talkingTimeLeft = 0; private talkingInterval: NodeJS.Timeout | null = null; @@ -50,7 +53,10 @@ export class GeminiSession implements VoiceAgentSession { ); } - async open( resumeHandle?: {} ) { + async open() { + if (this.opening) return; + this.opening = true; + const params: LiveConnectParameters = { model: GEMINI_MODEL, config: { @@ -74,14 +80,18 @@ export class GeminiSession implements VoiceAgentSession { proactivity: { proactiveAudio: true, }, - sessionResumption: resumeHandle + sessionResumption: { handle: this.previousHandle }, }, callbacks: { onmessage: (message) => this.onMessage(message), onerror: (e) => console.error('Gemini Error %o', e), onclose: (e) => { if (e.code !== 1000) { - console.error('Gemini Close: %o', e.reason); + console.error('Gemini Close: code=%d reason=%s', e.code, e.reason); + if (!this.closing && this.previousHandle) { + console.log('Attempting auto-reconnect with resumption...'); + this.reconnect(); + } } }, }, @@ -89,18 +99,23 @@ export class GeminiSession implements VoiceAgentSession { this.session = await this.genai.live.connect(params); - this.session.sendClientContent({ - turns: [ - { - text: 'introduce yourself', - }, - ], - turnComplete: true, - }); + if (!this.previousHandle) { + this.session.sendClientContent({ + turns: [ + { + text: 'introduce yourself', + }, + ], + turnComplete: true, + }); + } + if (this.talkingInterval) clearInterval(this.talkingInterval); this.talkingInterval = setInterval(() => { this.talkingTimeLeft = Math.max(this.talkingTimeLeft - 100, 0); }, 100); + + this.opening = false; } async close(wait: boolean) { @@ -110,14 +125,49 @@ export class GeminiSession implements VoiceAgentSession { await this.waitUntilDone(); } - if (this.talkingInterval) clearInterval(this.talkingInterval); - this.talkingInterval = null; + if (this.talkingInterval) { + clearInterval(this.talkingInterval); + this.talkingInterval = null; + } this.session?.close(); + this.session = null; this.closing = false; } - private onMessage(message: LiveServerMessage) { + private async reconnect() { + if (this.reconnecting || this.closing) return; + + this.reconnecting = true; + try { + if (this.session) { + this.session.close(); + this.session = null; + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + + await this.open(); + + this.sendContinuationPrompt(); + + console.log('Session reconnected successfully'); + } catch (err) { + console.error('Failed to reconnect:', err); + } finally { + this.reconnecting = false; + } + } + + private sendContinuationPrompt() { + if (!this.session) return; + this.session.sendClientContent({ + turns: [{ text: 'continue' }], + turnComplete: true, + }); + } + + private async onMessage(message: LiveServerMessage) { const transcription = message.serverContent?.outputTranscription?.text; if (transcription) { @@ -141,20 +191,7 @@ export class GeminiSession implements VoiceAgentSession { } if (message.sessionResumptionUpdate?.newHandle) { - console.log('Gemini session resumption handle updated: %s, starting new session', message.sessionResumptionUpdate.newHandle); - const newHandle = message.sessionResumptionUpdate.newHandle; - - try { - this.close(true); - } catch (e) { - console.error('Error closing Gemini session for resumption: %o', e); - } - try { - this.open({handle: newHandle}); - } - catch (e) { - console.error('Error opening Gemini session for resumption: %o', e); - } + this.previousHandle = message.sessionResumptionUpdate.newHandle; } message.toolCall?.functionCalls?.forEach((call) => { From 5bb2db0de30ea1a0c16d1f8cf0935e04a2a62679 Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Tue, 9 Dec 2025 17:45:28 +0100 Subject: [PATCH 3/9] stop game after timeout --- deep-sea-stories/packages/backend/src/game/room.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index ba7c0a4..d894612 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -27,6 +27,7 @@ export class GameRoom { private gameStarted: boolean = false; private gameSession: GameSession | null = null; private voiceAgentApi: VoiceAgentApi; + private gameTimeoutId: NodeJS.Timeout | null = null; constructor( fishjamClient: FishjamClient, @@ -173,10 +174,20 @@ export class GameRoom { type: 'gameStarted' as const, timestamp: Date.now(), }); + this.gameTimeoutId = setTimeout(async () => { + console.log( + `Game time limit reached for room ${this.roomId}, stopping game...`, + ); + await this.stopGame(true); + }, GAME_TIME_LIMIT_SECONDS * 1000); } async stopGame(wait: boolean = false) { console.log('Stopping game room %s', this.roomId); + if (this.gameTimeoutId) { + clearTimeout(this.gameTimeoutId); + this.gameTimeoutId = null; + } if (this.gameSession) { await this.gameSession.stopGame(wait); From e880af344650611e1cae3e222a4fbbe912f50baa Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Tue, 9 Dec 2025 18:00:07 +0100 Subject: [PATCH 4/9] announce game end --- .../packages/backend/src/agent/elevenlabs/session.ts | 4 ++++ .../packages/backend/src/agent/gemini/session.ts | 12 ++++++++++++ .../packages/backend/src/agent/session.ts | 1 + deep-sea-stories/packages/backend/src/game/room.ts | 7 ++++--- .../packages/backend/src/game/session.ts | 4 ++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts b/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts index d1f4876..7483acd 100644 --- a/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/elevenlabs/session.ts @@ -23,6 +23,10 @@ export class ElevenLabsSession implements VoiceAgentSession { this.audioInterface.setAgentAudioCallback(onAgentAudio); } + async announceTimeExpired() { + console.log('ElevenLabs session time expired (handled by platform)'); + } + async open() { await this.session.startSession(); } diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 885ae31..b1e55cf 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -47,6 +47,18 @@ export class GeminiSession implements VoiceAgentSession { this.onAgentAudio = onAgentAudio; } + async announceTimeExpired() { + if (!this.session) return; + + console.log('Sending time expired message to agent...'); + this.session.sendClientContent({ + turns: [{ + text: 'The game time has expired. Please tell the players that time is up, evaluate how close they were to solving the riddle, and then end the game by calling the endGame tool.' + }], + turnComplete: true, + }); + } + async waitUntilDone() { await new Promise((resolve) => setTimeout(resolve, this.talkingTimeLeft + 2000), diff --git a/deep-sea-stories/packages/backend/src/agent/session.ts b/deep-sea-stories/packages/backend/src/agent/session.ts index d60bdba..d9a7d7c 100644 --- a/deep-sea-stories/packages/backend/src/agent/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/session.ts @@ -2,6 +2,7 @@ export interface VoiceAgentSession { sendAudio: (audio: Buffer) => void; registerInterruptionCallback: (onInterrupt: () => void) => void; registerAgentAudioCallback: (onAgentAudio: (audio: Buffer) => void) => void; + announceTimeExpired: () => Promise; close: (wait: boolean) => Promise; open: () => Promise; } diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index d894612..e59d952 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -176,10 +176,11 @@ export class GameRoom { }); this.gameTimeoutId = setTimeout(async () => { console.log( - `Game time limit reached for room ${this.roomId}, stopping game...`, + `⏰ Game time limit reached for room ${this.roomId}`, ); - await this.stopGame(true); - }, GAME_TIME_LIMIT_SECONDS * 1000); + await this.gameSession?.announceTimeExpired(); + }, GAME_TIME_LIMIT_SECONDS * 1000, + ); } async stopGame(wait: boolean = false) { diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index cc62c85..69d2407 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -57,6 +57,10 @@ export class GameSession { await this.audioOrchestrator.start(); } + async announceTimeExpired() { + await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); + } + async stopGame(wait: boolean = false) { await this.audioOrchestrator.shutdown(wait); } From 8444bfa9dd104fa90a9526d349ea80a0794f2b2f Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Tue, 9 Dec 2025 18:12:11 +0100 Subject: [PATCH 5/9] change prompts to make sure it works correctly --- deep-sea-stories/packages/backend/src/agent/gemini/session.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index b1e55cf..25bb604 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -53,7 +53,7 @@ export class GeminiSession implements VoiceAgentSession { console.log('Sending time expired message to agent...'); this.session.sendClientContent({ turns: [{ - text: 'The game time has expired. Please tell the players that time is up, evaluate how close they were to solving the riddle, and then end the game by calling the endGame tool.' + text: 'IMPORTANT: The game time has expired. You must now: 1) Tell the players that time is up, 2) Evaluate how close they were to solving the riddle, 3) IMMEDIATELY call the endGame function to close the game session. Do not wait for player response - call endGame right after your message.' }], turnComplete: true, }); @@ -84,7 +84,7 @@ export class GeminiSession implements VoiceAgentSession { functionDeclarations: [ { name: 'endGame', - description: 'end the game', + description: 'Call this function to end the game session. You MUST call this when: 1) The players have correctly solved the riddle, 2) The game time has expired, 3) You are saying goodbye or ending the conversation. Always call this function after delivering your final message to players.', }, ], }, From 652b5029f880836661234cb0e7b6f233bb76d742 Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Tue, 9 Dec 2025 18:20:28 +0100 Subject: [PATCH 6/9] deafend agent when ending the game --- deep-sea-stories/packages/backend/src/game/session.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 69d2407..6e99d0f 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -58,6 +58,7 @@ export class GameSession { } async announceTimeExpired() { + this.setAiAgentMuted(true); await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); } From 55c7a970a9714ed5cb6e3ab8871f794d79d4059f Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Tue, 9 Dec 2025 18:21:24 +0100 Subject: [PATCH 7/9] format --- .../packages/backend/src/agent/gemini/session.ts | 13 ++++++++----- deep-sea-stories/packages/backend/src/game/room.ts | 7 ++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 25bb604..d151172 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -49,12 +49,14 @@ export class GeminiSession implements VoiceAgentSession { async announceTimeExpired() { if (!this.session) return; - + console.log('Sending time expired message to agent...'); this.session.sendClientContent({ - turns: [{ - text: 'IMPORTANT: The game time has expired. You must now: 1) Tell the players that time is up, 2) Evaluate how close they were to solving the riddle, 3) IMMEDIATELY call the endGame function to close the game session. Do not wait for player response - call endGame right after your message.' - }], + turns: [ + { + text: 'IMPORTANT: The game time has expired. You must now: 1) Tell the players that time is up, 2) Evaluate how close they were to solving the riddle, 3) IMMEDIATELY call the endGame function to close the game session. Do not wait for player response - call endGame right after your message.', + }, + ], turnComplete: true, }); } @@ -84,7 +86,8 @@ export class GeminiSession implements VoiceAgentSession { functionDeclarations: [ { name: 'endGame', - description: 'Call this function to end the game session. You MUST call this when: 1) The players have correctly solved the riddle, 2) The game time has expired, 3) You are saying goodbye or ending the conversation. Always call this function after delivering your final message to players.', + description: + 'Call this function to end the game session. You MUST call this when: 1) The players have correctly solved the riddle, 2) The game time has expired, 3) You are saying goodbye or ending the conversation. Always call this function after delivering your final message to players.', }, ], }, diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index e59d952..7ba7b42 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -175,12 +175,9 @@ export class GameRoom { timestamp: Date.now(), }); this.gameTimeoutId = setTimeout(async () => { - console.log( - `⏰ Game time limit reached for room ${this.roomId}`, - ); + console.log(`⏰ Game time limit reached for room ${this.roomId}`); await this.gameSession?.announceTimeExpired(); - }, GAME_TIME_LIMIT_SECONDS * 1000, - ); + }, GAME_TIME_LIMIT_SECONDS * 1000); } async stopGame(wait: boolean = false) { From e3ac5ecef5bcef033611cab332f9b879a0f93ad4 Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Tue, 9 Dec 2025 18:33:26 +0100 Subject: [PATCH 8/9] copilot suggestions --- .../packages/backend/src/agent/gemini/session.ts | 4 ++-- deep-sea-stories/packages/backend/src/game/room.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index d151172..335ff8e 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -151,7 +151,7 @@ export class GeminiSession implements VoiceAgentSession { } private async reconnect() { - if (this.reconnecting || this.closing) return; + if (this.opening || this.reconnecting || this.closing) return; this.reconnecting = true; try { @@ -182,7 +182,7 @@ export class GeminiSession implements VoiceAgentSession { }); } - private async onMessage(message: LiveServerMessage) { + private onMessage(message: LiveServerMessage) { const transcription = message.serverContent?.outputTranscription?.text; if (transcription) { diff --git a/deep-sea-stories/packages/backend/src/game/room.ts b/deep-sea-stories/packages/backend/src/game/room.ts index 7ba7b42..d298c35 100644 --- a/deep-sea-stories/packages/backend/src/game/room.ts +++ b/deep-sea-stories/packages/backend/src/game/room.ts @@ -176,7 +176,11 @@ export class GameRoom { }); this.gameTimeoutId = setTimeout(async () => { console.log(`⏰ Game time limit reached for room ${this.roomId}`); - await this.gameSession?.announceTimeExpired(); + try { + await this.gameSession?.announceTimeExpired(); + } catch (e) { + console.error('Error announcing time expired:', e); + } }, GAME_TIME_LIMIT_SECONDS * 1000); } From d8743638a25bd40d0d096737ea40a801c1e3bf29 Mon Sep 17 00:00:00 2001 From: Bernard Gawor Date: Fri, 12 Dec 2025 12:06:27 +0100 Subject: [PATCH 9/9] server deafen agent --- deep-sea-stories/packages/backend/src/agent/gemini/session.ts | 3 +++ deep-sea-stories/packages/backend/src/game/session.ts | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts index 335ff8e..06ecc6d 100644 --- a/deep-sea-stories/packages/backend/src/agent/gemini/session.ts +++ b/deep-sea-stories/packages/backend/src/agent/gemini/session.ts @@ -21,6 +21,7 @@ export class GeminiSession implements VoiceAgentSession { private closing = false; private opening = false; private reconnecting = false; + private ending = false; private talkingTimeLeft = 0; private talkingInterval: NodeJS.Timeout | null = null; @@ -31,6 +32,7 @@ export class GeminiSession implements VoiceAgentSession { } sendAudio(audio: Buffer) { + if (this.ending) return; this.session?.sendRealtimeInput({ audio: { data: audio.toString('base64'), @@ -49,6 +51,7 @@ export class GeminiSession implements VoiceAgentSession { async announceTimeExpired() { if (!this.session) return; + this.ending = true; console.log('Sending time expired message to agent...'); this.session.sendClientContent({ diff --git a/deep-sea-stories/packages/backend/src/game/session.ts b/deep-sea-stories/packages/backend/src/game/session.ts index 6e99d0f..69d2407 100644 --- a/deep-sea-stories/packages/backend/src/game/session.ts +++ b/deep-sea-stories/packages/backend/src/game/session.ts @@ -58,7 +58,6 @@ export class GameSession { } async announceTimeExpired() { - this.setAiAgentMuted(true); await this.audioOrchestrator.voiceAgentSession.announceTimeExpired(); }