Hands-Free, Eyes-Free Voice Interfaces

import { RealtimeClient } from '@openai/realtime-api-beta';

const client = new RealtimeClient({
  apiKey: process.env.OPENAI_API_KEY,
  model: 'gpt-realtime',
});

await client.connect();

// Configure session for accessibility
client.updateSession({
  voice: 'alloy',
  instructions: `You are an accessibility-first voice agent. 
Your goal is to make every interaction fully accessible without 
visual or tactile input.

ACCESSIBILITY RULES:
1. ALWAYS describe what you're doing ("I'm checking your balance now")
2. ALWAYS confirm actions before executing ("I'll transfer $100. Say 'confirm' to proceed")
3. ALWAYS provide context ("You have 3 accounts. Which would you like?")
4. NEVER assume user can see anything
5. NEVER rush the user (adaptive silence handling)
6. ALWAYS offer alternatives ("Or you can say 'go back'")
7. DESCRIBE numbers clearly ("$1,234.56 - that's one thousand, two hundred thirty-four dollars and fifty-six cents")
8. CONFIRM understanding ("Did I get that right?")

Pacing:
- Speak clearly, not too fast
- Pause between steps
- Let user interrupt anytime
- Never timeout before 30 seconds

Error handling:
- If you don't understand, ask clarifying question
- Never say "invalid input" - instead "I didn't catch that. Could you rephrase?"
- Offer examples of valid commands

Always maintain conversation state so user never has to repeat information.`,
  tools: [
    {
      type: 'function',
      name: 'check_balance',
      description: 'Check account balance',
      parameters: {
        type: 'object',
        properties: {
          account_id: { type: 'string', description: 'Account identifier to read balance from' }
        },
        required: ['account_id']
      }
    },
    {
      type: 'function',
      name: 'transfer_money',
      description: 'Transfer money between accounts',
      parameters: {
        type: 'object',
        properties: {
          from_account: { type: 'string', description: 'Source account identifier' },
          to_account: { type: 'string', description: 'Destination account identifier' },
          amount: { type: 'number', description: 'Transfer amount in account currency' }
        },
        required: ['from_account', 'to_account', 'amount']
      }
    }
  ]
});

// Handle tool calls with verbose feedback
client.on('conversation.item.created', async (event) => {
  const item = event.item;
  
  if (item.type === 'function_call') {
    const call = item.call;
    
    if (call.name === 'check_balance') {
      const accountId = JSON.parse(call.arguments).account_id;
      
      // Announce what you're doing
      client.sendUserMessageContent([{
        type: 'input_text',
        text: "I'm checking your balance now."
      }]);
      
      const balance = await getBalance(accountId);
      const account = await getAccountDetails(accountId);
      
      // Provide full context through function response
      client.realtime.send({
        type: 'conversation.item.create',
        item: {
          type: 'function_call_output',
          call_id: call.id,
          output: JSON.stringify({
            balance: balance,
            message: `Your ${account.type} account ending in ${account.last_four} ` +
              `has a balance of $${balance.toFixed(2)}. ` +
              `That's ${describeNumberVerbosely(balance)}. ` +
              `Would you like to transfer money, check another account, ` +
              `or do something else?`
          })
        }
      });
    }
    
    if (call.name === 'transfer_money') {
      const { from_account, to_account, amount } = JSON.parse(call.arguments);
      
      // Verbose confirmation before executing
      const fromDetails = await getAccountDetails(from_account);
      const toDetails = await getAccountDetails(to_account);
      
      // Send confirmation request
      client.sendUserMessageContent([{
        type: 'input_text',
        text: `I'll transfer $${amount.toFixed(2)} - that's ${describeNumberVerbosely(amount)} - ` +
          `from your ${fromDetails.type} account ending in ${fromDetails.last_four} ` +
          `to your ${toDetails.type} account ending in ${toDetails.last_four}. ` +
          `To confirm this transfer, please say "confirm transfer". ` +
          `Or say "cancel" if you'd like to change anything.`
      }]);
      
      // Note: Confirmation would be handled through conversation flow
      // The agent will wait for user response naturally
    }
  }
});

// Helper: Describe numbers verbosely for clarity
function describeNumberVerbosely(amount) {
  const dollars = Math.floor(amount);
  const cents = Math.round((amount - dollars) * 100);
  
  let description = '';
  
  if (dollars > 0) {
    description += `${numberToWords(dollars)} dollars`;
  }
  
  if (cents > 0) {
    if (dollars > 0) description += ' and ';
    description += `${cents} cents`;
  }
  
  return description;
}

function numberToWords(num) {
  // Convert 1234 to "one thousand two hundred thirty-four"
  // Implementation details omitted for brevity
}

// Help users navigate options without seeing them
client.on('conversation.updated', async ({ item, delta }) => {
  if (item.type === 'message' && item.role === 'user') {
    const userText = item.content[0]?.text;
    
    if (userText?.includes('list options') || userText?.includes('what are my options')) {
      const options = await getAvailableOptions();
      
      // Don't just list - give context and examples
      client.sendUserMessageContent([{
        type: 'input_text',
        text: `You have ${options.length} options. ` +
          `I'll describe each one. You can interrupt me anytime. ` +
          `Option 1: ${options[0].description}. ` +
          `To choose this, say "select option 1" or just say "${options[0].shortName}". ` +
          `Ready for option 2?`
      }]);
      
      // The agent will naturally wait for user response in the conversation flow
    }
  }
});

import asyncio
import json
from websocket import WebSocketApp

class AccessibleVoiceAgent:
    def __init__(self, api_key):
        self.api_key = api_key
        self.ws = None
        self.conversation_state = {}
    
    async def connect(self):
        """Connect to OpenAI Realtime API"""
        url = "wss://api.openai.com/v1/realtime?model=gpt-realtime"
        headers = [f"Authorization: Bearer {self.api_key}"]
        
        self.ws = WebSocketApp(
            url,
            header=headers,
            on_message=self.on_message,
            on_open=self.on_open
        )
        
    def on_open(self, ws):
        # Configure session for accessibility
        ws.send(json.dumps({
            "type": "session.update",
            "session": {
                "voice": "alloy",
                "instructions": "You are an accessibility-first voice agent...",
                "tools": [
                    {
                        "type": "function",
                        "name": "check_balance",
                        "description": "Check account balance",
                        "parameters": {
                            "type": "object",
                            "properties": {
                                "account_id": {"type": "string"}
                            }
                        }
                    }
                ]
            }
        }))
    
    def on_message(self, ws, message):
        event = json.loads(message)
        
        if event["type"] == "conversation.item.created":
            item = event["item"]
            
            if item.get("type") == "function_call":
                asyncio.create_task(self.handle_function_call(item))
    
    async def handle_function_call(self, item):
        """Handle function calls with verbose accessibility feedback"""
        call = item["call"]
        args = json.loads(call["arguments"])
        
        if call["name"] == "check_balance":
            account_id = args["account_id"]
            
            # Get data
            account = await get_account_details(account_id)
            balance = await get_balance(account_id)
            
            # Verbose response with full context
            response = {
                "balance": balance,
                "message": (
                    f"Your {account['type']} account ending in {account['last_four']} "
                    f"has a balance of ${balance:.2f}. "
                    f"That's {self.number_to_words(balance)} dollars. "
                    f"Would you like to transfer money, check another account, "
                    f"or do something else?"
                )
            }
            
            # Send function result
            self.ws.send(json.dumps({
                "type": "conversation.item.create",
                "item": {
                    "type": "function_call_output",
                    "call_id": call["id"],
                    "output": json.dumps(response)
                }
            }))
    
    async def transfer_money(self, from_id, to_id, amount):
        """
        Accessible money transfer with explicit verbal confirmation.
        Note: Confirmation happens through natural conversation flow,
        not programmatic waiting.
        """
        from_account = await get_account_details(from_id)
        to_account = await get_account_details(to_id)
        
        # Return detailed info for the agent to speak
        return {
            "from_account": from_account,
            "to_account": to_account,
            "amount": amount,
            "requires_confirmation": True,
            "confirmation_message": (
                f"I'll transfer ${amount:.2f} - that's {self.number_to_words(amount)} dollars - "
                f"from your {from_account['type']} account ending in {from_account['last_four']} "
                f"to your {to_account['type']} account ending in {to_account['last_four']}. "
                f"This will leave ${from_account['balance'] - amount:.2f} in your {from_account['type']} account. "
                f"To confirm, please say 'confirm transfer'. Or say 'cancel' to go back."
            )
        }
            return None
    
    def number_to_words(self, num):
        """
        Convert number to spoken words for accessibility
        Example: 1234.56 -> "one thousand two hundred thirty-four dollars and fifty-six cents"
        """
        # Implementation details...
        pass