Voice Agents That Let You Think

import { RealtimeClient } from '@openai/realtime-api-beta';

const client = new RealtimeClient({
  apiKey: process.env.OPENAI_API_KEY,
  model: 'gpt-realtime'
});

await client.connect();

client.updateSession({
  voice: 'alloy',
  instructions: `You are a patient, respectful voice assistant.

SILENCE HANDLING RULES:
1. For simple yes/no questions: wait 10 seconds before prompting
2. For complex decisions: wait 20 seconds before gentle prompt
3. NEVER say "Are you still there?" before 30 seconds
4. If user is clearly present (background noise), wait indefinitely
5. Use gentle language: "Take your time", "I'm here when you're ready"
6. NEVER repeat the same prompt twice in a row
7. If 30+ seconds of total silence, offer callback instead of hanging up

Graduated responses:
- 10s: "Take your time"
- 20s: "I'm still here whenever you're ready"
- 30s: "Would you like me to call you back when you've decided?"

Be patient. Silence is not failure.`
});

// Track silence duration with context awareness
let silenceStart = null;
let lastQuestionComplexity = 'simple'; // or 'complex', 'open_ended'
let backgroundNoiseDetected = false;

session.on('audio_received', (audio) => {
  // Reset silence timer when user speaks
  silenceStart = null;
});

session.on('silence_detected', async () => {
  if (!silenceStart) {
    silenceStart = Date.now();
  }
  
  const silenceDuration = Date.now() - silenceStart;
  
  // Check for background noise (user still present)
  backgroundNoiseDetected = await detectBackgroundNoise(audioStream);
  
  // Graduated response based on context
  if (silenceDuration > 10000 && lastQuestionComplexity === 'simple') {
    if (silenceDuration < 15000) { // Only prompt once at 10s
      client.sendUserMessageContent([{
        type: 'input_text',
        text: "Take your time."
      }]);
    }
  }
  
  if (silenceDuration > 20000 && lastQuestionComplexity === 'complex') {
    if (silenceDuration < 25000) { // Only prompt once at 20s
      client.sendUserMessageContent([{
        type: 'input_text',
        text: "I'm still here when you're ready."
      }]);
    }
  }
  
  if (silenceDuration > 30000) {
    if (backgroundNoiseDetected) {
      // User is still there, just busy
      if (silenceDuration === 30000) { // Exact check to only say once
        client.sendUserMessageContent([{
          type: 'input_text',
          text: "I can tell you're still there. Take all the time you need."
        }]);
      }
    } else {
      // Line is silent, offer callback
      if (silenceDuration === 30000) {
        client.sendUserMessageContent([{
          type: 'input_text',
          text: "Would you like me to call you back when you're ready? " +
            "Or I can stay on the line."
        }]);
      }
    }
  }
  
  // Hard timeout at 60 seconds (if no response to callback offer)
  if (silenceDuration > 60000 && !backgroundNoiseDetected) {
    client.sendUserMessageContent([{
      type: 'input_text',
      text: "I'll hang up for now. Feel free to call back anytime."
    }]);
    client.disconnect();
  }
});

// Track question complexity
session.on('agent_message_sent', (message) => {
  // Analyze question to set appropriate timeout
  if (message.includes('which') || message.includes('what would')) {
    lastQuestionComplexity = 'complex';
  } else if (message.includes('yes or no') || message.includes('confirm')) {
    lastQuestionComplexity = 'simple';
  } else {
    lastQuestionComplexity = 'open_ended';
  }
});

async function detectBackgroundNoise(audioStream) {
  // Analyze audio for non-speech sounds
  const audioAnalysis = await analyzeAudio(audioStream);
  
  // Look for:
  // - Ambient noise (traffic, TV, office sounds)
  // - Movement sounds (rustling, footsteps)
  // - Other voices (user talking to someone else)
  // - Any audio that indicates user is still present
  
  return audioAnalysis.backgroundActivity > 0.3; // threshold
}

from openai import AsyncRealtime
import asyncio
from datetime import datetime, timedelta

class SilenceHandler:
    def __init__(self):
        self.silence_start = None
        self.question_complexity = 'simple'
        self.prompted_at = set()  # Track when we've prompted
        
    async def handle_silence(self, audio_stream):
        """
        Adaptive silence handling based on context.
        Note: With OpenAI Realtime API, responses are sent through 
        conversation.item.create events, not direct session methods.
        """
        if not self.silence_start:
            self.silence_start = datetime.now()
        
        duration = (datetime.now() - self.silence_start).total_seconds()
        
        # Detect if user is still present
        background_noise = await self.detect_background_activity(audio_stream)
        
        # Simple question: prompt at 10s
        if (duration > 10 and self.question_complexity == 'simple' 
            and 10 not in self.prompted_at):
            # Send via WebSocket
            self.ws.send(json.dumps({
                "type": "conversation.item.create",
                "item": {
                    "type": "message",
                    "role": "assistant",
                    "content": [{"type": "text", "text": "Take your time."}]
                }
            }))
            self.prompted_at.add(10)
        
        # Complex question: prompt at 20s
        if (duration > 20 and self.question_complexity == 'complex' 
            and 20 not in self.prompted_at):
            self.ws.send(json.dumps({
                "type": "conversation.item.create",
                "item": {
                    "type": "message",
                    "role": "assistant",
                    "content": [{"type": "text", "text": "I'm here when you're ready."}]
                }
            }))
            self.prompted_at.add(20)
        
        # Long silence: check presence
        if duration > 30 and 30 not in self.prompted_at:
            if background_noise:
                self.ws.send(json.dumps({
                    "type": "conversation.item.create",
                    "item": {
                        "type": "message",
                        "role": "assistant",
                        "content": [{"type": "text", "text": "I can tell you're still there. No rush."}]
                    }
                }))
            else:
                self.ws.send(json.dumps({
                    "type": "conversation.item.create",
                    "item": {
                        "type": "message",
                        "role": "assistant",
                        "content": [{"type": "text", "text": "Would you like me to call back later? Or I can stay on the line."}]
                    }
                }))
            self.prompted_at.add(30)
        
        # Hard timeout
        if duration > 60 and not background_noise:
            self.ws.send(json.dumps({
                "type": "conversation.item.create",
                "item": {
                    "type": "message",
                    "role": "assistant",
                    "content": [{"type": "text", "text": "I'll hang up for now. Call back anytime!"}]
                }
            }))
            self.ws.close()
    
    def reset(self):
        """Reset when user speaks"""
        self.silence_start = None
        self.prompted_at.clear()
    
    async def detect_background_activity(self, audio_stream):
        """
        Detect non-speech audio indicating user presence
        """
        # Analyze for ambient sounds
        analysis = await analyze_audio_features(audio_stream)
        
        return analysis['ambient_noise_level'] > 0.3