Voice Agents With Emotional Intelligence: Match User Emotion In Real Time

Text: "This is fine"

Agent: "I can help you rebook. Let me check available flights."

const adaptResponseToEmotion = (transcript, emotionData) => {
  const { sentiment, intensity, urgency } = emotionData;
  
  if (sentiment === 'distressed' && urgency === 'high') {
    return {
      tone: 'empathetic',
      pace: 'slower',
      response: "I understand this is really stressful, especially with the wedding tomorrow. Let me find you the next available flight right now—I'll prioritize getting you there on time."
    };
  } else if (sentiment === 'neutral' || sentiment === 'calm') {
    return {
      tone: 'professional',
      pace: 'normal',
      response: "I can help you rebook. Let me check available flights for you."
    };
  }
};

def analyze_pitch(audio_features):
    pitch_mean = audio_features['pitch_mean']
    pitch_variance = audio_features['pitch_variance']
    
    if pitch_mean > 250 and pitch_variance > 50:
        return 'anxious_or_excited'
    elif pitch_mean < 150 and pitch_variance < 20:
        return 'calm_or_defeated'
    else:
        return 'neutral'

def analyze_speech_rate(transcript, duration):
    words_per_minute = (len(transcript.split()) / duration) * 60
    
    if words_per_minute > 180:
        return 'urgent_or_anxious'
    elif words_per_minute < 100:
        return 'hesitant_or_uncertain'
    else:
        return 'normal'

def analyze_volume(audio_features):
    volume_peaks = audio_features['volume_peaks']
    
    if max(volume_peaks) > 85:
        return 'frustrated_or_angry'
    elif max(volume_peaks) < 50:
        return 'resigned_or_sad'
    else:
        return 'normal'

class EmotionDetector:
    def detect_emotion(self, audio_features, transcript, duration):
        pitch_signal = self.analyze_pitch(audio_features)
        rate_signal = self.analyze_speech_rate(transcript, duration)
        volume_signal = self.analyze_volume(audio_features)
        
        # Decision tree
        if volume_signal == 'frustrated_or_angry' and rate_signal == 'urgent_or_anxious':
            return {'emotion': 'angry', 'confidence': 0.85, 'urgency': 'high'}
        elif pitch_signal == 'anxious_or_excited' and rate_signal == 'urgent_or_anxious':
            return {'emotion': 'anxious', 'confidence': 0.80, 'urgency': 'high'}
        elif pitch_signal == 'calm_or_defeated' and volume_signal == 'resigned_or_sad':
            return {'emotion': 'defeated', 'confidence': 0.75, 'urgency': 'medium'}
        elif rate_signal == 'hesitant_or_uncertain':
            return {'emotion': 'confused', 'confidence': 0.70, 'urgency': 'low'}
        else:
            return {'emotion': 'neutral', 'confidence': 0.90, 'urgency': 'low'}

class EmotionallyIntelligentAgent {
  constructor(openaiClient) {
    this.client = openaiClient;
  }
  
  async respondWithEmpathy(userEmotion, userMessage) {
    const systemPrompt = this.buildEmpatheticPrompt(userEmotion);
    
    const response = await this.client.chat.completions.create({
      model: 'gpt-4',
      messages: [
        { role: 'system', content: systemPrompt },
        { role: 'user', content: userMessage }
      ],
      temperature: 0.7
    });
    
    return response.choices[0].message.content;
  }
  
  buildEmpatheticPrompt(emotion) {
    const prompts = {
      angry: `
You're speaking with someone who is frustrated and upset.
- Acknowledge their frustration immediately
- Apologize if appropriate
- Offer a concrete solution quickly
- Keep responses concise
- Use calming, slower pace

Example: "I completely understand your frustration. Let me fix this for you right now."
`,
      anxious: `
You're speaking with someone who is worried or stressed.
- Reassure them early in your response
- Be specific about next steps
- Give realistic timelines
- Avoid adding uncertainty

Example: "I understand this is stressful. I'm going to help you right now, and we'll have this sorted in the next few minutes."
`,
      defeated: `
You're speaking with someone who feels resigned or discouraged.
- Show extra care and patience
- Avoid cheerful platitudes
- Offer genuine help
- Take ownership of the problem

Example: "I hear you. This shouldn't have happened. Let me see what I can do to make this right."
`,
      confused: `
You're speaking with someone who doesn't understand something.
- Slow down your explanation
- Use simpler language
- Ask if they need clarification
- Be patient with follow-up questions

Example: "Let me break this down step by step. First, [simple explanation]. Does that make sense so far?"
`,
      neutral: `
You're speaking with someone who is calm and direct.
- Be professional and efficient
- Provide clear, concise answers
- Move at a normal pace

Example: "I can help with that. Let me pull up your information."
`
    };
    
    return prompts[emotion] || prompts.neutral;
  }
}

class EmotionTracker:
    def __init__(self):
        self.emotion_history = []
    
    def track_emotion(self, timestamp, emotion, confidence):
        self.emotion_history.append({
            'timestamp': timestamp,
            'emotion': emotion,
            'confidence': confidence
        })
    
    def detect_emotion_shift(self):
        if len(self.emotion_history) < 2:
            return None
        
        prev_emotion = self.emotion_history[-2]['emotion']
        curr_emotion = self.emotion_history[-1]['emotion']
        
        if prev_emotion != curr_emotion:
            return {
                'from': prev_emotion,
                'to': curr_emotion,
                'shift_type': self.classify_shift(prev_emotion, curr_emotion)
            }
        return None
    
    def classify_shift(self, from_emotion, to_emotion):
        # Positive shifts
        if from_emotion in ['angry', 'anxious', 'defeated'] and to_emotion == 'neutral':
            return 'de-escalation'
        
        # Negative shifts
        if from_emotion == 'neutral' and to_emotion in ['angry', 'anxious']:
            return 'escalation'
        
        return 'lateral'

# Usage
tracker = EmotionTracker()

# Start of call
tracker.track_emotion(0, 'angry', 0.85)

# After empathetic response
tracker.track_emotion(30, 'anxious', 0.75)

# After solution offered
tracker.track_emotion(60, 'neutral', 0.90)

shift = tracker.detect_emotion_shift()
# Returns: {'from': 'anxious', 'to': 'neutral', 'shift_type': 'de-escalation'}

const shouldEscalate = (emotionTracker, currentEmotion) => {
  const history = emotionTracker.getHistory();
  
  // Escalate if:
  // 1. User has been angry for >2 minutes
  if (currentEmotion === 'angry' && 
      history.filter(e => e.emotion === 'angry').length > 4) {
    return {
      escalate: true,
      reason: 'Sustained frustration',
      priority: 'high'
    };
  }
  
  // 2. Emotion escalated from neutral to angry
  const recentShift = emotionTracker.detectEmotionShift();
  if (recentShift?.shift_type === 'escalation') {
    return {
      escalate: true,
      reason: 'Negative emotion shift',
      priority: 'medium'
    };
  }
  
  // 3. User explicitly requests human
  // (detected in transcript, not just emotion)
  
  return { escalate: false };
};

Angry user: "I hear that you're frustrated, and I want to make sure you get the best help. Let me connect you to a specialist who can resolve this immediately."

Anxious user: "I want to make sure this gets handled perfectly. Let me bring in a team member who specializes in this exact situation."

class EmotionalIntelligenceMetrics:
    def __init__(self):
        self.de_escalations = 0
        self.escalations = 0
        self.emotion_detection_accuracy = []
    
    def log_conversation(self, start_emotion, end_emotion, resolution):
        if start_emotion in ['angry', 'anxious', 'defeated'] and end_emotion == 'neutral':
            self.de_escalations += 1
        elif start_emotion == 'neutral' and end_emotion in ['angry', 'anxious']:
            self.escalations += 1
        
        return {
            'de_escalation_rate': self.de_escalations / (self.de_escalations + self.escalations),
            'resolution': resolution
        }

import { OpenAI } from 'openai';

class EmpatheticVoiceAgent {
  constructor() {
    this.client = new OpenAI();
    this.emotionTracker = new EmotionTracker();
    this.detector = new EmotionDetector();
  }
  
  async handleUserInput(audioStream, timestamp) {
    // Get transcription + audio features
    const result = await this.client.audio.transcriptions.create({
      file: audioStream,
      model: 'whisper-1',
      response_format: 'verbose_json'
    });
    
    // Detect emotion
    const emotion = this.detector.detect_emotion(
      result.audio_features,
      result.text,
      result.duration
    );
    
    // Track over time
    this.emotionTracker.track_emotion(timestamp, emotion.emotion, emotion.confidence);
    
    // Check for escalation
    const escalation = shouldEscalate(this.emotionTracker, emotion.emotion);
    if (escalation.escalate) {
      return this.handoffToHuman(escalation.reason);
    }
    
    // Generate empathetic response
    const agent = new EmotionallyIntelligentAgent(this.client);
    const response = await agent.respondWithEmpathy(emotion.emotion, result.text);
    
    return {
      response,
      emotion: emotion.emotion,
      confidence: emotion.confidence
    };
  }
  
  handoffToHuman(reason) {
    return {
      action: 'escalate',
      message: "I want to make sure you get the best help. Let me connect you to a specialist.",
      reason
    };
  }
}