Verify Identity By Speaking

import { RealtimeClient } from '@openai/realtime-api-beta';
const voicePrintAPI = require('./voiceprint-service'); // 3rd party biometrics

const client = new RealtimeClient({
  apiKey: process.env.OPENAI_API_KEY,
  model: 'gpt-realtime'
});

await client.connect();

client.updateSession({
  voice: 'alloy',
  instructions: `You are a voice banking assistant with biometric authentication.

When a user calls:
1. Greet them naturally (don't mention biometrics explicitly)
2. Capture their initial speech for voiceprint analysis
3. If voiceprint matches with high confidence (>85%), 
   address them by name and ask ONE secondary question
4. If voiceprint has medium confidence (50-85%), 
   ask TWO verification questions
5. If voiceprint doesn't match or confidence is low (<50%), 
   use standard authentication flow

Be conversational and natural. Never make users feel like 
they're being scanned or analyzed.`
});

// Capture initial speech for voiceprint
let voiceprintCaptured = false;
let audioBuffer = [];

// Listen for audio input
client.on('conversation.item.input_audio_transcription.completed', async (event) => {
  if (!voiceprintCaptured) {
    // In production, you'd capture raw audio data
    // For this example, we simulate voiceprint analysis
    
    // After sufficient speech, analyze voiceprint
    if (audioBuffer.length >= 15) { // ~3-5 seconds at 20ms chunks
      const voiceprint = await voicePrintAPI.extract(audioBuffer);
      const match = await voicePrintAPI.compare(
        voiceprint,
        currentUser.storedVoiceprint
      );
      
      voiceprintCaptured = true;
      
      // Route based on confidence
      if (match.confidence > 0.85) {
        session.setContext({
          authLevel: 'high_confidence',
          userName: currentUser.firstName,
          requireSecondaryCheck: true,
          secondaryQuestions: 1
        });
      } else if (match.confidence > 0.50) {
        session.setContext({
          authLevel: 'medium_confidence',
          requireSecondaryCheck: true,
          secondaryQuestions: 2
        });
      } else {
        session.setContext({
          authLevel: 'low_confidence',
          requireStandardAuth: true
        });
      }
    }
  }
});

// Handle secondary verification
session.on('function_call', async (call) => {
  if (call.name === 'verify_secondary_info') {
    const { question, answer } = call.arguments;
    
    const isCorrect = await verifyAnswer(
      currentUser.id,
      question,
      answer
    );
    
    if (isCorrect) {
      await logAuthentication({
        userId: currentUser.id,
        method: 'voice_biometric + secondary',
        confidence: match.confidence,
        timestamp: new Date().toISOString()
      });
      
      return {
        authenticated: true,
        message: "Authentication successful"
      };
    } else {
      return {
        authenticated: false,
        message: "Answer doesn't match our records"
      };
    }
  }
});

// Example using a voice biometrics service
async function extractVoiceprint(audioBuffer) {
  const features = {
    mfcc: extractMFCC(audioBuffer), // Mel-frequency cepstral coefficients
    pitch: analyzePitch(audioBuffer),
    formants: extractFormants(audioBuffer),
    cadence: analyzeCadence(audioBuffer)
  };
  
  const voiceprint = await ml_model.encode(features);
  return voiceprint; // 512-dimensional vector
}

async function compareVoiceprints(current, stored) {
  const similarity = cosineSimilarity(current, stored);
  
  return {
    confidence: similarity,
    match: similarity > 0.70, // threshold for match
    requiresSecondaryCheck: similarity < 0.85
  };
}

from openai import AsyncRealtime
import numpy as np
from voiceprint import VoiceprintAnalyzer

analyzer = VoiceprintAnalyzer()

async def authenticate_with_voice(session, user_id):
    """
    Authenticate user using voice biometrics
    """
    # Capture initial speech (3-5 seconds)
    audio_buffer = await session.capture_audio(duration=5.0)
    
    # Extract voiceprint
    current_voiceprint = analyzer.extract(audio_buffer)
    
    # Load stored voiceprint
    stored_voiceprint = await load_voiceprint(user_id)
    
    # Compare
    similarity = np.dot(current_voiceprint, stored_voiceprint)
    confidence = float(similarity)
    
    if confidence > 0.85:
        # High confidence - fast track
        user = await load_user(user_id)
        # Send response via WebSocket
        ws.send(json.dumps({
            "type": "conversation.item.create",
            "item": {
                "type": "message",
                "role": "assistant",
                "content": [{
                    "type": "text",
                    "text": f"Hi {user.first_name}, I recognize your voice. " +
                           f"For security, can you confirm your date of birth?"
                }]
            }
        }))
        
        # In production, verification happens through conversation flow
        # The agent will naturally handle the response
        await log_auth(user_id, 'voice_biometric', confidence)
        return True
    
    elif confidence > 0.50:
        # Medium confidence - extra verification
        ws.send(json.dumps({
            "type": "conversation.item.create",
            "item": {
                "type": "message",
                "role": "assistant",
                "content": [{
                    "type": "text",
                    "text": "For security, I need to verify two pieces of information. " +
                           "First, what's your date of birth?"
                }]
            }
        }))
        
        # Verification continues through natural conversation
        # Agent will ask for PIN as second verification
        await log_auth(user_id, 'voice_biometric_medium', confidence)
        return True
    
    else:
        # Low confidence - standard authentication
        ws.send(json.dumps({
            "type": "conversation.item.create",
            "item": {
                "type": "message",
                "role": "assistant",
                "content": [{
                    "type": "text",
                    "text": "I don't recognize this voice. Let's verify your identity. " +
                           "Please provide your account number."
                }]
            }
        }))
        
        return await standard_auth_flow(ws, user_id)

import { RealtimeClient } from '@openai/realtime-api-beta';

async function enrollVoiceprint(userId) {
  const client = new RealtimeClient({
    apiKey: process.env.OPENAI_API_KEY,
    model: 'gpt-realtime'
  });
  
  await client.connect();
  
  client.updateSession({
    voice: 'alloy',
    instructions: `You're helping a user enroll their voice for biometric authentication.

Say: "To set up voice authentication, I need you to read this phrase three times. Ready?"

Then provide this phrase: "My voice is my secure password for authentication."

After they read it three times, confirm enrollment is complete.`
  });
  
  const recordings = [];
  
  // In production, capture audio through conversation events
  // This is a simplified example
  client.on('conversation.item.input_audio_transcription.completed', async (event) => {
    if (event.transcript.includes("my voice is my secure password")) {
      // Store this audio sample for voiceprint extraction
      recordings.push(event);
      
      if (recordings.length < 3) {
        client.sendUserMessageContent([{
          type: 'input_text',
          text: `Great! Please read it ${3 - recordings.length} more time(s).`
        }]);
      }
    }
  });
  
  // Wait for all three recordings
  await waitForRecordings(recordings, 3);
  
  // Extract voiceprint from all three recordings
  const voiceprints = recordings.map(r => extractVoiceprint(r));
  
  // Average them for robustness
  const masterVoiceprint = averageVoiceprints(voiceprints);
  
  // Store
  await storeVoiceprint(userId, masterVoiceprint);
  
  client.disconnect();
  
  return {
    success: true,
    message: "Voice authentication enrolled successfully"
  };
}

async function detectLiveness(audioStream) {
  // Check for:
  // - Background noise patterns (recordings are too clean)
  // - Micro-variations in pitch (humans vary, recordings don't)
  // - Response timing (humans pause naturally)
  
  const livenessScore = await liveness_model.analyze(audioStream);
  return livenessScore > 0.80; // threshold for "real human"
}

const authLevels = {
  viewing_balance: 'voice_only',
  making_transfer: 'voice + security_question',
  changing_password: 'voice + OTP_to_registered_device'
};

if (match.confidence < 0.50) {
  // Fallback to standard auth
  client.sendUserMessageContent([{
    type: 'input_text',
    text: "I'm having trouble verifying your voice. " +
          "Let's use standard verification instead."
  }]);
  return await standardAuthFlow(client);
}

// Check age of voiceprint
if (daysSinceEnrollment > 365) {
  client.sendUserMessageContent([{
    type: 'input_text',
    text: "It's been a while since we updated your voice profile. " +
          "Would you like to refresh it? It only takes 30 seconds."
  }]);
}