Voice Agents With Perfect Memory: Context That Persists Across Turns

User: "I want to order a large pepperoni pizza"
Agent: "Large pepperoni pizza. Got it."

[2 minutes later]

User: "Actually, remove the olives"
Agent: "I don't see any order. What would you like?"

User: "I want to order a large pepperoni pizza"
Agent: "Large pepperoni pizza. Got it."

[2 minutes later]

User: "Actually, remove the olives"
Agent: "Got it, removing olives from your large pepperoni pizza."

class ConversationMemory {
  constructor(maxTurns = 10) {
    this.turns = [];
    this.entities = new Map();  // Track mentioned entities
    this.maxTurns = maxTurns;
  }
  
  addTurn(speaker, message, entities = {}) {
    const turn = {
      speaker,
      message,
      timestamp: Date.now(),
      entities
    };
    
    this.turns.push(turn);
    
    // Update entity tracking
    Object.entries(entities).forEach(([key, value]) => {
      this.entities.set(key, value);
    });
    
    // Keep only recent turns
    if (this.turns.length > this.maxTurns) {
      this.turns.shift();
    }
    
    return turn;
  }
  
  getRecentTurns(count = 5) {
    return this.turns.slice(-count);
  }
  
  getEntity(entityKey) {
    return this.entities.get(entityKey);
  }
  
  getFullContext() {
    return {
      recent_turns: this.getRecentTurns(),
      entities: Object.fromEntries(this.entities),
      turn_count: this.turns.length
    };
  }
}

// Usage
const memory = new ConversationMemory();

// Turn 1
memory.addTurn('user', 'I want to order a large pepperoni pizza', {
  order_type: 'pizza',
  size: 'large',
  topping: 'pepperoni'
});

memory.addTurn('agent', 'Large pepperoni pizza. Got it.');

// Turn 2 (2 minutes later)
memory.addTurn('user', 'Actually, remove the olives');

// Agent can now reference previous order
const context = memory.getFullContext();
console.log(context.entities);
// { order_type: 'pizza', size: 'large', topping: 'pepperoni' }

"Remove IT"
"Change THAT to medium"
"What was THE PRICE again?"

class ReferenceResolver {
  constructor(conversationMemory) {
    this.memory = conversationMemory;
  }
  
  resolveReferences(currentMessage) {
    const resolved = { ...currentMessage };
    
    // Resolve "it"
    if (/\bit\b/i.test(currentMessage.text)) {
      const lastEntity = this.getLastMentionedEntity();
      if (lastEntity) {
        resolved.text = currentMessage.text.replace(/\bit\b/i, lastEntity.name);
        resolved.resolved_reference = {
          original: 'it',
          refers_to: lastEntity
        };
      }
    }
    
    // Resolve "that"
    if (/\bthat\b/i.test(currentMessage.text)) {
      const lastObject = this.getLastMentionedObject();
      if (lastObject) {
        resolved.text = currentMessage.text.replace(/\bthat\b/i, lastObject.name);
        resolved.resolved_reference = {
          original: 'that',
          refers_to: lastObject
        };
      }
    }
    
    return resolved;
  }
  
  getLastMentionedEntity() {
    const recentTurns = this.memory.getRecentTurns(3);
    
    for (let i = recentTurns.length - 1; i >= 0; i--) {
      const turn = recentTurns[i];
      if (turn.entities && Object.keys(turn.entities).length > 0) {
        const entityKey = Object.keys(turn.entities)[0];
        return {
          name: entityKey,
          value: turn.entities[entityKey]
        };
      }
    }
    
    return null;
  }
}

// Usage
const resolver = new ReferenceResolver(memory);

const userMessage = { text: "Remove it" };
const resolved = resolver.resolveReferences(userMessage);

console.log(resolved);
// {
//   text: "Remove olives",
//   resolved_reference: {
//     original: 'it',
//     refers_to: { name: 'topping', value: 'olives' }
//   }
// }

class SessionSummarizer:
    def __init__(self, openai_client):
        self.client = openai_client
    
    def summarize_conversation(self, turns):
        # Convert turns to text
        conversation_text = '\n'.join([
            f"{turn['speaker']}: {turn['message']}"
            for turn in turns
        ])
        
        # Generate summary
        response = self.client.chat.completions.create(
            model='gpt-4',
            messages=[
                {
                    'role': 'system',
                    'content': '''
Summarize this conversation in 2-3 sentences. Include:
- Main topic/request
- Key decisions made
- Current state

Format: {"topic": "...", "decisions": [...], "state": "..."}
'''
                },
                { 'role': 'user', 'content': conversation_text }
            ],
            response_format={ 'type': 'json_object' }
        )
        
        return json.loads(response.choices[0].message.content)

# Example
summarizer = SessionSummarizer(openai_client)

summary = summarizer.summarize_conversation([
    {'speaker': 'user', 'message': 'I want to order a large pepperoni pizza'},
    {'speaker': 'agent', 'message': 'Large pepperoni pizza. Got it.'},
    {'speaker': 'user', 'message': 'Add extra cheese'},
    {'speaker': 'agent', 'message': 'Added extra cheese'},
    {'speaker': 'user', 'message': 'Remove the olives'},
    {'speaker': 'agent', 'message': 'Removed olives'}
])

# Returns:
# {
#   "topic": "Pizza order",
#   "decisions": ["Large size", "Pepperoni", "Extra cheese", "No olives"],
#   "state": "Order ready for checkout"
# }

class PersistentUserMemory {
  constructor(database) {
    this.db = database;
  }
  
  async getUserPreferences(userId) {
    return await this.db.query(
      'SELECT * FROM user_preferences WHERE user_id = ?',
      [userId]
    );
  }
  
  async saveConversationSummary(userId, sessionId, summary) {
    await this.db.query(
      'INSERT INTO conversation_history (user_id, session_id, summary, timestamp) VALUES (?, ?, ?, ?)',
      [userId, sessionId, JSON.stringify(summary), Date.now()]
    );
  }
  
  async getPreviousSessionContext(userId, limit = 3) {
    const sessions = await this.db.query(
      'SELECT summary FROM conversation_history WHERE user_id = ? ORDER BY timestamp DESC LIMIT ?',
      [userId, limit]
    );
    
    return sessions.map(s => JSON.parse(s.summary));
  }
}

// Usage
const persistentMemory = new PersistentUserMemory(database);

// Start new session
const userId = 'user_123';
const previousContext = await persistentMemory.getPreviousSessionContext(userId);

console.log(previousContext);
// [
//   { topic: "Pizza order", decisions: ["Large", "Pepperoni"], state: "Delivered" },
//   { topic: "Account update", decisions: ["Changed email"], state: "Complete" }
// ]

// Agent can now say:
// "Hi! Last time you ordered a large pepperoni pizza. Same again?"

class PizzaOrderAgent {
  constructor() {
    this.memory = new ConversationMemory();
    this.resolver = new ReferenceResolver(this.memory);
    this.order = null;
  }
  
  async processMessage(userMessage) {
    // Resolve references
    const resolved = this.resolver.resolveReferences({ text: userMessage });
    
    // Extract intent
    const intent = await this.detectIntent(resolved.text);
    
    // Process based on intent
    let response;
    if (intent === 'new_order') {
      response = await this.createOrder(resolved.text);
    } else if (intent === 'modify_order') {
      response = await this.modifyOrder(resolved.text);
    } else if (intent === 'confirm_order') {
      response = await this.confirmOrder();
    }
    
    // Store turn
    this.memory.addTurn('user', userMessage);
    this.memory.addTurn('agent', response);
    
    return response;
  }
  
  async createOrder(message) {
    // Parse order details
    this.order = {
      size: this.extractSize(message),
      toppings: this.extractToppings(message)
    };
    
    this.memory.entities.set('current_order', this.order);
    
    return `${this.order.size} pizza with ${this.order.toppings.join(', ')}. Got it.`;
  }
  
  async modifyOrder(message) {
    if (!this.order) {
      return "I don't see an order yet. What would you like?";
    }
    
    // Check for remove/add keywords
    if (message.includes('remove')) {
      const topping = this.extractToppings(message)[0];
      this.order.toppings = this.order.toppings.filter(t => t !== topping);
      return `Removed ${topping} from your ${this.order.size} pizza.`;
    } else if (message.includes('add')) {
      const topping = this.extractToppings(message)[0];
      this.order.toppings.push(topping);
      return `Added ${topping} to your ${this.order.size} pizza.`;
    }
    
    return "I'm not sure what to change. Can you be more specific?";
  }
  
  async confirmOrder() {
    if (!this.order) {
      return "I don't see an order to confirm.";
    }
    
    return `Your order: ${this.order.size} pizza with ${this.order.toppings.join(', ')}. 
Total: $${this.calculatePrice()}. 
Shall I place the order?`;
  }
}

// Conversation flow
const agent = new PizzaOrderAgent();

console.log(await agent.processMessage("I want a large pepperoni pizza"));
// "Large pizza with pepperoni. Got it."

console.log(await agent.processMessage("Add extra cheese"));
// "Added extra cheese to your large pizza."

console.log(await agent.processMessage("Remove the olives"));
// "Removed olives from your large pizza."

console.log(await agent.processMessage("What's the total?"));
// "Your order: large pizza with pepperoni, extra cheese. Total: $18. Shall I place the order?"

class MemoryMetrics:
    def __init__(self):
        self.references_resolved = 0
        self.references_failed = 0
        self.context_maintained = 0
        self.context_lost = 0
    
    def log_reference_resolution(self, success):
        if success:
            self.references_resolved += 1
        else:
            self.references_failed += 1
    
    def log_context_maintenance(self, maintained):
        if maintained:
            self.context_maintained += 1
        else:
            self.context_lost += 1
    
    def get_metrics(self):
        return {
            'reference_resolution_rate': 
                self.references_resolved / (self.references_resolved + self.references_failed),
            'context_maintenance_rate':
                self.context_maintained / (self.context_maintained + self.context_lost)
        }

# Target metrics:
# Reference resolution: >85%
# Context maintenance: >90%

class ProductionConversationMemory extends ConversationMemory {
  constructor() {
    super();
    this.currentTopic = null;
  }
  
  detectTopicShift(newMessage) {
    // Simple topic detection
    const topics = ['order', 'account', 'billing', 'support'];
    
    for (const topic of topics) {
      if (newMessage.toLowerCase().includes(topic)) {
        if (this.currentTopic && this.currentTopic !== topic) {
          // Topic shifted
          this.clearMemory();
          this.currentTopic = topic;
          return true;
        }
        this.currentTopic = topic;
        break;
      }
    }
    
    return false;
  }
  
  clearMemory() {
    // Save summary before clearing
    const summary = this.getSummary();
    
    // Clear turns but keep important entities
    this.turns = [];
    
    return summary;
  }
}