Skip The Menu, Just Talk: Intent-Based Routing For Voice Agents

"I need help with my order"
→ Is that a new order?
→ Is it about tracking?
→ Is it a return?
→ Is it a refund?

// User: "My package was supposed to arrive yesterday but it's not here"

const sessionConfig = {
  modalities: ['audio', 'text'],
  instructions: `
You are a delivery support agent. Listen for:
- Delivery delays or missing packages
- Damaged items
- Wrong items received
- Address issues

When you detect intent, route immediately. Don't ask "which department do you need?"

For delivery issues:
1. Pull recent orders
2. Check tracking status
3. Provide update or solution
`,
  tools: [
    {
      type: 'function',
      name: 'check_order_status',
      description: 'Look up order status by customer ID',
      parameters: {
        type: 'object',
        properties: {
          customer_id: { type: 'string' }
        }
      }
    }
  ]
};

from openai import OpenAI

client = OpenAI()

def handle_delivery_issue(customer_id):
    # Intent detected: delivery issue
    order_status = check_order_status(customer_id)
    
    if order_status['status'] == 'delayed':
        return {
            'action': 'provide_update',
            'message': f"I see your order is delayed. It's currently in {order_status['location']} and should arrive by tomorrow evening. I can offer expedited shipping or a partial refund—which would help?",
            'options': ['expedite', 'refund']
        }
    elif order_status['status'] == 'delivered':
        return {
            'action': 'check_delivery_location',
            'message': "The tracking shows it was delivered yesterday at 3pm. Was it left in a specific location? Sometimes packages are placed in safe spots."
        }

User: "I want to return this shirt, but also I was charged twice"

const detectMultipleIntents = (transcript) => {
  const intents = [];
  
  // Check for return keywords + context
  if (transcript.includes('return') || transcript.includes('send back')) {
    intents.push({
      type: 'return_request',
      confidence: 0.9,
      keywords: ['return', 'shirt']
    });
  }
  
  // Check for billing keywords + context
  if (transcript.includes('charged twice') || transcript.includes('double charge')) {
    intents.push({
      type: 'billing_error',
      confidence: 0.95,
      keywords: ['charged twice']
    });
  }
  
  return intents;
};

// Agent response:
// "I can help with both. Let me start with the double charge—I'll refund that right now while we set up your return. Does that work?"

User: "I need to change my account"

def clarify_ambiguous_intent(transcript, context):
    if 'change' in transcript and 'account' in transcript:
        # Check recent activity for clues
        recent_actions = get_recent_activity(context['user_id'])
        
        if recent_actions['last_action'] == 'failed_login':
            # Likely password issue
            return "Are you trying to reset your password?"
        elif recent_actions['payment_failed']:
            # Likely payment method
            return "Do you need to update your payment method?"
        else:
            # Generic clarification
            return "What would you like to change—your email, password, or payment info?"

const handleUncertainIntent = (intent, confidence) => {
  if (confidence > 0.8) {
    // High confidence: act immediately
    return routeToAgent(intent);
  } else if (confidence > 0.5) {
    // Medium confidence: confirm first
    return confirmIntent(intent);
  } else {
    // Low confidence: ask open-ended question
    return "I want to make sure I help with the right thing. Can you tell me more about what you need?"
  }
};

User: "I want to cancel my subscription"
Agent: "I can help with that. Can I ask why you're canceling?"
User: "It's too expensive"
Agent: "Would a 20% discount work instead?"
User: "Actually, yeah, that would help"

class IntentTracker:
    def __init__(self):
        self.history = []
    
    def update_intent(self, new_intent):
        self.history.append(new_intent)
        
        # If user shifts from cancel to accept offer
        if self.history[-2] == 'cancel_subscription' and new_intent == 'accept_discount':
            return {
                'action': 'apply_discount',
                'route_to': 'retention_team',
                'priority': 'high'
            }

class VoiceIntentRouter {
  constructor(openaiClient) {
    this.client = openaiClient;
    this.intentMap = {
      'delivery_issue': { team: 'fulfillment', priority: 'high' },
      'billing_error': { team: 'payments', priority: 'urgent' },
      'product_question': { team: 'support', priority: 'medium' },
      'return_request': { team: 'returns', priority: 'medium' },
      'cancel_subscription': { team: 'retention', priority: 'high' }
    };
  }
  
  async detectIntent(audioStream) {
    const response = await this.client.audio.transcriptions.create({
      file: audioStream,
      model: 'whisper-1',
      response_format: 'verbose_json'
    });
    
    const transcript = response.text;
    const intent = await this.classifyIntent(transcript);
    
    return {
      transcript,
      intent: intent.label,
      confidence: intent.score,
      routing: this.intentMap[intent.label]
    };
  }
  
  async classifyIntent(transcript) {
    const completion = await this.client.chat.completions.create({
      model: 'gpt-4',
      messages: [
        {
          role: 'system',
          content: `
Classify user intent into one of these categories:
- delivery_issue
- billing_error
- product_question
- return_request
- cancel_subscription

Return JSON: {"label": "category", "score": 0.0-1.0}
`
        },
        { role: 'user', content: transcript }
      ],
      response_format: { type: 'json_object' }
    });
    
    return JSON.parse(completion.choices[0].message.content);
  }
  
  routeToAgent(intent) {
    const routing = this.intentMap[intent];
    
    if (!routing) {
      throw new Error(`Unknown intent: ${intent}`);
    }
    
    return {
      team: routing.team,
      priority: routing.priority,
      transfer_message: `Connecting you to our ${routing.team} team who can help immediately.`
    };
  }
}

// Usage
const router = new VoiceIntentRouter(openaiClient);

const result = await router.detectIntent(userAudioStream);
console.log(`Detected: ${result.intent} (${result.confidence})`);

if (result.confidence > 0.7) {
  const route = router.routeToAgent(result.intent);
  console.log(`Routing to ${route.team} (${route.priority} priority)`);
}

import json

class IntentMetrics:
    def __init__(self):
        self.correct = 0
        self.incorrect = 0
        self.unclear = 0
    
    def log_intent_result(self, predicted, actual, confidence):
        result = {
            'predicted': predicted,
            'actual': actual,
            'confidence': confidence,
            'correct': predicted == actual
        }
        
        if predicted == actual:
            self.correct += 1
        elif actual == 'unclear':
            self.unclear += 1
        else:
            self.incorrect += 1
        
        return result
    
    def get_accuracy(self):
        total = self.correct + self.incorrect + self.unclear
        return {
            'accuracy': self.correct / total if total > 0 else 0,
            'unclear_rate': self.unclear / total if total > 0 else 0,
            'error_rate': self.incorrect / total if total > 0 else 0
        }

# Target: >85% accuracy, <10% unclear, <5% error