Add Voice Agents To Phone Lines: Telephony Integration Pattern with Twilio

// Install Twilio SDK
// npm install twilio

const twilio = require('twilio');
const accountSid = process.env.TWILIO_ACCOUNT_SID;
const authToken = process.env.TWILIO_AUTH_TOKEN;
const client = twilio(accountSid, authToken);

// Configure your phone number
const PHONE_NUMBER = '+18005551234';  // Your Twilio number

// Set webhook URL for incoming calls
await client.incomingPhoneNumbers
  .list({phoneNumber: PHONE_NUMBER})
  .then(numbers => {
    numbers[0].update({
      voiceUrl: 'https://your-server.com/voice/incoming',
      voiceMethod: 'POST'
    });
  });

const express = require('express');
const VoiceResponse = require('twilio').twiml.VoiceResponse;

const app = express();

app.post('/voice/incoming', async (req, res) => {
  const twiml = new VoiceResponse();
  
  // Greet caller
  twiml.say({
    voice: 'Polly.Joanna'
  }, 'Thank you for calling. Connecting you to our assistant...');
  
  // Connect to voice agent via WebSocket
  const connect = twiml.connect();
  connect.stream({
    url: `wss://your-server.com/voice/stream`,
    parameters: {
      callSid: req.body.CallSid,
      from: req.body.From,
      to: req.body.To
    }
  });
  
  res.type('text/xml');
  res.send(twiml.toString());
});

app.listen(3000, () => {
  console.log('Telephony server running on port 3000');
});

const WebSocket = require('ws');
const wss = new WebSocket.Server({ port: 8080 });

wss.on('connection', async (ws) => {
  console.log('New call connected');
  
  // Store call metadata
  let callData = {
    sid: null,
    from: null,
    streamSid: null
  };
  
  // Connect to OpenAI Realtime API
  const openaiWs = new WebSocket('wss://api.openai.com/v1/realtime', {
    headers: {
      'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
      'OpenAI-Beta': 'realtime=v1'
    }
  });
  
  // When Twilio sends audio
  ws.on('message', async (message) => {
    const msg = JSON.parse(message);
    
    switch (msg.event) {
      case 'start':
        callData.sid = msg.start.callSid;
        callData.from = msg.start.customParameters.from;
        callData.streamSid = msg.start.streamSid;
        
        // Initialize OpenAI session
        openaiWs.send(JSON.stringify({
          type: 'session.update',
          session: {
            model: 'gpt-realtime',
            voice: 'alloy',
            instructions: getAgentInstructions(callData.from),
            tools: getAvailableTools()
          }
        }));
        break;
      
      case 'media':
        // Forward audio from Twilio to OpenAI
        openaiWs.send(JSON.stringify({
          type: 'input_audio_buffer.append',
          audio: msg.media.payload  // base64 encoded audio
        }));
        break;
      
      case 'stop':
        console.log('Call ended');
        openaiWs.close();
        break;
    }
  });
  
  // When OpenAI responds with audio
  openaiWs.on('message', (message) => {
    const msg = JSON.parse(message);
    
    if (msg.type === 'response.audio.delta') {
      // Forward audio from OpenAI to Twilio
      ws.send(JSON.stringify({
        event: 'media',
        streamSid: callData.streamSid,
        media: {
          payload: msg.delta  // base64 encoded audio
        }
      }));
    }
    
    // Handle function calls (like escalation)
    if (msg.type === 'response.function_call_arguments.done') {
      handleFunctionCall(msg, callData, ws);
    }
  });
  
  ws.on('close', () => {
    openaiWs.close();
  });
});

function getAgentInstructions(callerNumber) {
  return `You are a customer support voice agent answering phone calls.

CAPABILITIES:
- Answer common questions about products, billing, and account
- Look up order status
- Process simple requests
- Escalate to human when needed

WHEN TO ESCALATE:
- Complex technical issues
- Billing disputes requiring human judgment
- Sensitive account changes
- Customer explicitly requests human
- You're uncertain about the answer

PHONE ETIQUETTE:
- Be warm but professional
- Keep responses concise (phone audio context)
- Confirm you heard correctly before acting
- Don't make the customer repeat themselves
- If escalating, explain why and assure them context will be preserved

Current caller: ${callerNumber}
`;
}

function getAvailableTools() {
  return [
    {
      type: 'function',
      name: 'lookup_order_status',
      description: 'Look up the status of a customer order',
      parameters: {
        type: 'object',
        properties: {
          order_number: {
            type: 'string',
            description: 'The order number'
          }
        },
        required: ['order_number']
      }
    },
    {
      type: 'function',
      name: 'escalate_to_human',
      description: 'Escalate call to human agent',
      parameters: {
        type: 'object',
        properties: {
          reason: {
            type: 'string',
            description: 'Why escalation is needed'
          },
          summary: {
            type: 'string',
            description: 'Summary of conversation so far'
          },
          urgency: {
            type: 'string',
            enum: ['low', 'medium', 'high'],
            description: 'Urgency level'
          }
        },
        required: ['reason', 'summary']
      }
    }
  ];
}

async function handleFunctionCall(msg, callData, twilioWs) {
  const functionName = msg.name;
  const args = JSON.parse(msg.arguments);
  
  if (functionName === 'escalate_to_human') {
    console.log(`[Escalation] Call ${callData.sid}: ${args.reason}`);
    
    // Log context to CRM
    await logToContextCRM({
      callSid: callData.sid,
      from: callData.from,
      summary: args.summary,
      reason: args.reason,
      urgency: args.urgency,
      timestamp: Date.now()
    });
    
    // Tell customer we're transferring
    await sendAudioToTwilio(twilioWs, callData.streamSid, 
      "Let me connect you to a specialist who can help better. " +
      "They'll have all the context from our conversation."
    );
    
    // Transfer call to human agent
    await transferToHuman(callData.sid, {
      queue: args.urgency === 'high' ? 'priority' : 'general',
      context: args.summary,
      reason: args.reason
    });
  }
  
  if (functionName === 'lookup_order_status') {
    // Handle order lookup
    const orderStatus = await lookupOrder(args.order_number);
    
    // Send result back to OpenAI
    openaiWs.send(JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'function_call_output',
        call_id: msg.call_id,
        output: JSON.stringify(orderStatus)
      }
    }));
  }
}

async function transferToHuman(callSid, options) {
  // Update call to connect to human agent queue
  await client.calls(callSid).update({
    twiml: `
      <Response>
        <Say>Transferring now.</Say>
        <Enqueue workflowSid="${WORKFLOW_SID}">
          <Task priority="${options.urgency === 'high' ? 10 : 5}">
            {
              "context": "${options.context}",
              "reason": "${options.reason}",
              "from_agent": true
            }
          </Task>
        </Enqueue>
      </Response>
    `
  });
}

async function logToContextCRM(data) {
  // Log to CRM so human agent can see full context
  await crmAPI.createInteraction({
    customer_phone: data.from,
    channel: 'phone',
    agent_type: 'voice_ai',
    summary: data.summary,
    escalation_reason: data.reason,
    urgency: data.urgency,
    call_sid: data.callSid,
    timestamp: data.timestamp
  });
}

from fastapi import FastAPI, WebSocket, Request
from twilio.twiml.voice_response import VoiceResponse, Connect
import asyncio
import websockets
import json

app = FastAPI()

@app.post("/voice/incoming")
async def handle_incoming_call(request: Request):
    """Handle incoming phone call from Twilio."""
    form_data = await request.form()
    
    response = VoiceResponse()
    
    # Greet caller
    response.say(
        "Thank you for calling. Connecting you to our assistant.",
        voice="Polly.Joanna"
    )
    
    # Connect to media stream
    connect = Connect()
    connect.stream(
        url=f"wss://{request.url.hostname}/voice/stream",
        parameters={
            'call_sid': form_data['CallSid'],
            'from': form_data['From'],
            'to': form_data['To']
        }
    )
    response.append(connect)
    
    return Response(content=str(response), media_type="text/xml")

@app.websocket("/voice/stream")
async def voice_stream(websocket: WebSocket):
    """Bridge Twilio media stream to OpenAI Realtime API."""
    await websocket.accept()
    
    # Connect to OpenAI
    async with websockets.connect(
        'wss://api.openai.com/v1/realtime',
        extra_headers={
            'Authorization': f'Bearer {OPENAI_API_KEY}',
            'OpenAI-Beta': 'realtime=v1'
        }
    ) as openai_ws:
        
        # Handle bidirectional audio streaming
        async def twilio_to_openai():
            """Forward audio from Twilio to OpenAI."""
            async for message in websocket.iter_text():
                msg = json.loads(message)
                
                if msg['event'] == 'start':
                    # Initialize OpenAI session
                    await openai_ws.send(json.dumps({
                        'type': 'session.update',
                        'session': {
                            'model': 'gpt-realtime',
                            'voice': 'alloy',
                            'instructions': get_instructions(),
                            'tools': get_tools()
                        }
                    }))
                
                elif msg['event'] == 'media':
                    # Forward audio
                    await openai_ws.send(json.dumps({
                        'type': 'input_audio_buffer.append',
                        'audio': msg['media']['payload']
                    }))
        
        async def openai_to_twilio():
            """Forward audio from OpenAI to Twilio."""
            async for message in openai_ws:
                msg = json.loads(message)
                
                if msg['type'] == 'response.audio.delta':
                    # Forward audio
                    await websocket.send_json({
                        'event': 'media',
                        'media': {
                            'payload': msg['delta']
                        }
                    })
                
                elif msg['type'] == 'response.function_call_arguments.done':
                    # Handle function calls
                    await handle_function_call(msg, websocket)
        
        # Run both directions concurrently
        await asyncio.gather(
            twilio_to_openai(),
            openai_to_twilio()
        )

def get_instructions() -> str:
    return """You are a customer support voice agent on a phone call.

Handle common questions, look up orders, and escalate when needed.

Be warm, professional, and concise. Phone context requires brevity.

Escalate to humans for:
- Complex issues
- Sensitive matters
- Customer requests
- Uncertainty

When escalating, explain why and assure context preservation."""

def get_tools() -> list:
    return [
        {
            'type': 'function',
            'name': 'lookup_order_status',
            'description': 'Look up order status',
            'parameters': {
                'type': 'object',
                'properties': {
                    'order_number': {'type': 'string'}
                },
                'required': ['order_number']
            }
        },
        {
            'type': 'function',
            'name': 'escalate_to_human',
            'description': 'Escalate to human agent',
            'parameters': {
                'type': 'object',
                'properties': {
                    'reason': {'type': 'string'},
                    'summary': {'type': 'string'},
                    'urgency': {
                        'type': 'string',
                        'enum': ['low', 'medium', 'high']
                    }
                },
                'required': ['reason', 'summary']
            }
        }
    ]

async def handle_function_call(msg: dict, websocket: WebSocket):
    """Handle function calls from voice agent."""
    function_name = msg['name']
    args = json.loads(msg['arguments'])
    
    if function_name == 'escalate_to_human':
        # Log context
        await log_to_crm({
            'summary': args['summary'],
            'reason': args['reason'],
            'urgency': args['urgency']
        })
        
        # Initiate transfer
        await transfer_call(websocket, args)
    
    elif function_name == 'lookup_order_status':
        # Look up order
        status = await lookup_order(args['order_number'])
        
        # Return to agent
        # Send back to OpenAI...

async function identifyCaller(phoneNumber) {
  // Look up caller in CRM
  const customer = await crm.findByPhone(phoneNumber);
  
  if (customer) {
    // Personalized greeting
    return `Welcome back, ${customer.firstName}! How can I help you today?`;
  }
  
  return "Thank you for calling. How can I help you today?";
}

function detectLanguage(firstUtterance) {
  // Detect language from first few words
  const lang = languageDetector.detect(firstUtterance);
  
  // Switch voice agent language
  openaiWs.send(JSON.stringify({
    type: 'session.update',
    session: {
      instructions: getInstructions(lang),
      voice: getVoiceForLanguage(lang)
    }
  }));
}

async function logCallAnalytics(callData) {
  await analytics.track('phone_call_completed', {
    call_sid: callData.sid,
    duration: callData.duration,
    resolution: callData.resolution,  // 'agent' or 'escalated'
    topics: callData.topics,
    sentiment: callData.sentiment,
    customer_phone: callData.from
  });
}