Call Any Business With A Voice Agent

// Instead of this:
const reservation = await fetch('https://api.restaurant.com/bookings', {
  method: 'POST',
  body: JSON.stringify({
    party_size: 4,
    date: '2025-04-15',
    time: '19:00'
  })
});

// You do this:
const reservation = await voiceAgent.call({
  phone: '+1-555-RESTAURANT',
  task: 'Book a table for 4 on April 15 at 7pm',
  expected_duration: '90 seconds'
});

import { RealtimeClient } from '@openai/realtime-api-beta';
import twilio from 'twilio';

class VoiceAgentPhoneBridge {
  constructor() {
    this.twilioClient = twilio(
      process.env.TWILIO_ACCOUNT_SID,
      process.env.TWILIO_AUTH_TOKEN
    );
    this.realtimeClient = new RealtimeClient({ 
      apiKey: process.env.OPENAI_API_KEY 
    });
  }

  async callBusiness(phoneNumber, task, context = {}) {
    console.log(`Calling ${phoneNumber} to: ${task}`);
    
    // Start Twilio outbound call
    const call = await this.twilioClient.calls.create({
      from: process.env.TWILIO_PHONE_NUMBER,
      to: phoneNumber,
      url: `${process.env.BASE_URL}/voice-webhook`,
      statusCallback: `${process.env.BASE_URL}/call-status`,
      record: true, // Record for debugging
      timeout: 60 // Hang up if no answer after 60 seconds
    });

    // Connect Realtime API to call audio
    await this.realtimeClient.connect();
    
    // Configure voice agent for this specific task
    await this.realtimeClient.updateSession({
      instructions: `
You are calling a business on behalf of a user.

Task: ${task}
Context: ${JSON.stringify(context)}

Guidelines:
1. Be polite and professional
2. State your purpose clearly and early
3. If you reach an IVR, navigate it efficiently
4. If you reach voicemail, leave a clear message with callback number
5. Extract all relevant information
6. Confirm details before ending call
7. End call politely once task is complete

Return structured data:
{
  "success": true|false,
  "result": <extracted information>,
  "conversation_summary": "<brief summary>",
  "call_duration": <seconds>
}
`,
      voice: 'alloy',
      modalities: ['audio'],
      turn_detection: { type: 'server_vad', threshold: 0.5 }
    });

    // Track call progress
    const callData = {
      callSid: call.sid,
      startTime: Date.now(),
      transcript: [],
      result: null
    };

    // Handle conversation
    this.realtimeClient.on('conversation.item.completed', (event) => {
      callData.transcript.push({
        role: event.item.role,
        content: event.item.formatted.transcript || event.item.formatted.text,
        timestamp: Date.now() - callData.startTime
      });

      // Check if agent indicated task completion
      if (event.item.role === 'assistant' && 
          event.item.formatted.text.includes('TASK_COMPLETE')) {
        this.endCall(call.sid, callData);
      }
    });

    // Wait for call to complete
    return new Promise((resolve) => {
      this.twilioClient.calls(call.sid)
        .on('completed', () => {
          callData.endTime = Date.now();
          resolve(this.extractStructuredResult(callData));
        });
    });
  }

  async extractStructuredResult(callData) {
    // Use GPT-4 to extract structured data from conversation
    const response = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        model: 'gpt-4',
        messages: [{
          role: 'user',
          content: `
Extract structured information from this phone call transcript:

${JSON.stringify(callData.transcript, null, 2)}

Return JSON with:
{
  "success": <did task complete successfully>,
  "result": <key information extracted>,
  "issues": <any problems encountered>,
  "follow_up_needed": <true if human needs to call back>
}
`
        }],
        response_format: { type: 'json_object' }
      })
    });

    const data = await response.json();
    return {
      ...JSON.parse(data.choices[0].message.content),
      call_duration: (callData.endTime - callData.startTime) / 1000,
      transcript: callData.transcript
    };
  }

  async endCall(callSid, callData) {
    // Agent completed task, hang up politely
    await this.realtimeClient.sendUserMessageContent([{
      type: 'text',
      text: 'Thank the person and end the call now.'
    }]);

    // Give agent 3 seconds to say goodbye, then force hang up
    setTimeout(async () => {
      await this.twilioClient.calls(callSid).update({ status: 'completed' });
    }, 3000);
  }
}

// Usage examples
const bridge = new VoiceAgentPhoneBridge();

// Example 1: Book dentist appointment
const appointment = await bridge.callBusiness(
  '+1-555-DENTIST',
  'Book a teeth cleaning appointment',
  {
    patient_name: 'John Smith',
    preferred_dates: ['April 15', 'April 16', 'April 17'],
    preferred_times: ['morning', 'early afternoon'],
    insurance: 'Delta Dental'
  }
);

console.log('Appointment result:', appointment);
// {
//   success: true,
//   result: {
//     date: '2025-04-16',
//     time: '10:30 AM',
//     confirmation_number: 'DEN-4421',
//     location: '123 Main St'
//   },
//   call_duration: 87
// }

// Example 2: Check car repair status
const repairStatus = await bridge.callBusiness(
  '+1-555-AUTOSHOP',
  'Check repair status for vehicle',
  {
    customer_name: 'Jane Doe',
    vehicle: '2020 Toyota Camry',
    ticket_number: 'REP-8821'
  }
);

console.log('Repair status:', repairStatus);
// {
//   success: true,
//   result: {
//     status: 'ready for pickup',
//     total_cost: '$450',
//     work_completed: 'Brake pad replacement, oil change',
//     pickup_hours: 'Mon-Fri 8am-6pm, Sat 9am-2pm'
//   },
//   call_duration: 62
// }

// Example 3: Place catering order
const cateringOrder = await bridge.callBusiness(
  '+1-555-RESTAURANT',
  'Place catering order for office lunch',
  {
    company: 'Acme Corp',
    date: 'April 20',
    time: '12:00 PM',
    headcount: 25,
    budget: '$15 per person',
    dietary: 'vegetarian options needed'
  }
);

console.log('Catering order:', cateringOrder);
// {
//   success: true,
//   result: {
//     menu: 'Mixed sandwich platter + sides',
//     total_cost: '$375',
//     delivery_time: '11:45 AM',
//     order_number: 'CAT-9432'
//   },
//   call_duration: 124
// }

// Detect IVR vs human
this.realtimeClient.on('audio.started', async (event) => {
  const audioAnalysis = await this.analyzeAudio(event.audio);
  
  if (audioAnalysis.is_ivr) {
    // Switch to DTMF mode
    await this.navigateIVR(audioAnalysis.menu_options);
  } else {
    // Continue with conversational mode
    await this.conversationalMode();
  }
});

async navigateIVR(menuOptions) {
  // Parse options: "Press 1 for appointments, 2 for billing..."
  const targetOption = this.matchOptionToTask(this.task, menuOptions);
  
  // Send DTMF tone
  await this.twilioClient.calls(this.callSid).update({
    method: 'POST',
    twiml: `<Response><Play digits="${targetOption}"/></Response>`
  });
}

this.realtimeClient.on('conversation.item.completed', (event) => {
  const transcript = event.item.formatted.transcript;
  
  // Detect voicemail patterns
  if (this.isVoicemail(transcript)) {
    this.leaveVoicemail();
  }
});

isVoicemail(transcript) {
  const voicemailIndicators = [
    'leave a message',
    'after the beep',
    'press pound',
    'record your message',
    'not available'
  ];
  
  return voicemailIndicators.some(indicator => 
    transcript.toLowerCase().includes(indicator)
  );
}

async leaveVoicemail() {
  await this.realtimeClient.sendUserMessageContent([{
    type: 'text',
    text: `
This is a voicemail. Leave a clear, professional message:
- State your name and purpose
- Provide callback number: ${process.env.CALLBACK_NUMBER}
- Keep it under 30 seconds
- Speak clearly
`
  }]);
}

this.realtimeClient.on('conversation.item.completed', (event) => {
  const response = event.item.formatted.transcript;
  
  // Detect confusion signals
  if (this.isConfused(response)) {
    this.attemptClarification();
  }
  
  // Detect hard failures
  if (this.isHardFailure(response)) {
    this.escalateToHuman();
  }
});

isConfused(response) {
  const confusionSignals = [
    "i don't understand",
    "can you repeat",
    "what do you mean",
    "i'm not sure"
  ];
  
  return confusionSignals.some(signal => 
    response.toLowerCase().includes(signal)
  );
}

async attemptClarification() {
  await this.realtimeClient.sendUserMessageContent([{
    type: 'text',
    text: 'Rephrase your request more simply and directly.'
  }]);
  
  this.clarificationAttempts++;
  
  if (this.clarificationAttempts > 2) {
    this.escalateToHuman();
  }
}

async escalateToHuman() {
  // Agent can't complete task, needs human help
  await this.twilioClient.calls(this.callSid).update({
    url: `${process.env.BASE_URL}/human-handoff`
  });
  
  // Notify operations team
  await this.notifyOperations({
    phone: this.phoneNumber,
    task: this.task,
    transcript: this.transcript,
    reason: 'Clarification failed after 2 attempts'
  });
}