Use Meta-Prompts To Build Voice State Machines

// Turn 1
User: "I need to book a flight"
Agent: "Where to?"

// Turn 3
User: "Also can you check my rewards balance?"
Agent: "Um... let me help with that flight first..."

// Turn 6
User: "Wait, what dates did I say again?"
Agent: [Lost context, starts over]

conversation_states = {
    "GREETING": ["ask_intent"],
    "BOOKING": ["gather_dates", "gather_destination", "confirm"],
    "REWARDS": ["check_balance", "explain_tiers"],
    "COMPLETE": ["thank_user", "close"]
}

// Step 1: Describe the task
const taskDescription = `
Help users research and compare health insurance plans.
They need to: understand plan types, compare costs, 
check doctor networks, and complete enrollment.
`;

// Step 2: Meta-prompt generates state machine
const metaPrompt = `
Given this task: "${taskDescription}"
Generate a conversation state machine with:
- Clear states for each phase
- Transition rules between states
- Validation checkpoints
- Error recovery paths
`;

// Step 3: AI outputs structured state machine
const generatedStateMachine = await generateStateMachine(metaPrompt);

{
  "states": {
    "INTRO": {
      "description": "Explain available plan types",
      "next": ["NEEDS_ASSESSMENT", "DIRECT_COMPARE"]
    },
    "NEEDS_ASSESSMENT": {
      "description": "Ask about coverage needs, budget, doctor preferences",
      "validates": ["age", "household_size", "budget_range"],
      "next": ["PLAN_RECOMMENDATION", "DOCTOR_SEARCH"]
    },
    "PLAN_RECOMMENDATION": {
      "description": "Show 2-3 matching plans with cost breakdown",
      "next": ["DEEP_DIVE", "DOCTOR_CHECK", "ENROLL"]
    },
    "DOCTOR_CHECK": {
      "description": "Verify specific doctors are in network",
      "next": ["PLAN_RECOMMENDATION", "ENROLL"]
    },
    "ENROLL": {
      "description": "Confirm selection and complete enrollment",
      "validates": ["plan_id", "payment_method"],
      "next": ["COMPLETE"]
    },
    "COMPLETE": {
      "description": "Provide confirmation and next steps",
      "terminal": true
    }
  },
  "error_recovery": {
    "confused": "Return to NEEDS_ASSESSMENT",
    "missing_info": "Stay in current state, prompt for specific field",
    "changed_mind": "Allow jump to INTRO or PLAN_RECOMMENDATION"
  }
}

import { RealtimeClient } from '@openai/realtime-api-beta';

class MetaPromptStateMachine {
  constructor() {
    this.client = new RealtimeClient({ apiKey: process.env.OPENAI_API_KEY });
  }

  async generateStateMachine(taskDescription) {
    // Meta-prompt that generates structured state machine
    const metaPrompt = `
You are a conversation flow architect. Given a task description, generate a JSON state machine for a voice agent conversation.

Task: ${taskDescription}

Generate a state machine with:
1. States: Each phase of the conversation
2. Transitions: Valid next states from each state
3. Validation: Required data at each state
4. Error recovery: What to do when things go wrong

Output only valid JSON. No explanations.

Format:
{
  "states": {
    "STATE_NAME": {
      "description": "What happens in this state",
      "validates": ["field1", "field2"],
      "next": ["NEXT_STATE1", "NEXT_STATE2"]
    }
  },
  "error_recovery": {
    "situation": "recovery_action"
  }
}
`;

    // Generate state machine with GPT-4
    const response = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        model: 'gpt-4',
        messages: [{ role: 'user', content: metaPrompt }],
        response_format: { type: 'json_object' }
      })
    });

    const data = await response.json();
    return JSON.parse(data.choices[0].message.content);
  }

  async runVoiceAgent(stateMachine, taskDescription) {
    let currentState = Object.keys(stateMachine.states)[0]; // Start at first state
    const collectedData = {};

    // Configure Realtime API session
    await this.client.connect();
    
    // Set up voice agent with state-aware instructions
    await this.client.updateSession({
      instructions: `
You are a helpful voice assistant. 
Task: ${taskDescription}

You are currently in state: ${currentState}
${stateMachine.states[currentState].description}

Track what information you collect. When you have all required data 
for this state, transition to the next appropriate state.
`,
      voice: 'alloy',
      modalities: ['text', 'audio'],
      input_audio_transcription: { model: 'whisper-1' }
    });

    // Handle state transitions
    this.client.on('conversation.item.completed', async (event) => {
      const state = stateMachine.states[currentState];
      
      // Check if we have all required validations
      if (state.validates) {
        const missingFields = state.validates.filter(f => !collectedData[f]);
        if (missingFields.length > 0) {
          console.log(`Still need: ${missingFields.join(', ')}`);
          return; // Stay in current state
        }
      }

      // All validations passed, determine next state
      const userIntent = await this.detectIntent(event.item.content);
      const nextState = this.chooseNextState(currentState, userIntent, state.next);
      
      if (nextState) {
        console.log(`Transitioning: ${currentState} → ${nextState}`);
        currentState = nextState;
        
        // Update agent instructions for new state
        await this.client.updateSession({
          instructions: `
State: ${currentState}
${stateMachine.states[currentState].description}

Previously collected: ${JSON.stringify(collectedData)}
`
        });
      }
    });

    // Start conversation
    await this.client.sendUserMessageContent([
      { type: 'text', text: 'Start conversation' }
    ]);
  }

  async detectIntent(content) {
    // Use fast intent classification
    const response = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        model: 'gpt-3.5-turbo',
        messages: [{
          role: 'user',
          content: `Extract intent from: "${content}". 
                   Return one of: needs_help, knows_what_they_want, 
                   has_question, ready_to_proceed, wants_to_change`
        }],
        max_tokens: 10
      })
    });

    const data = await response.json();
    return data.choices[0].message.content.trim();
  }

  chooseNextState(currentState, userIntent, possibleNext) {
    // Map intents to state transitions
    const intentToState = {
      'needs_help': possibleNext.find(s => s.includes('ASSESSMENT')),
      'knows_what_they_want': possibleNext.find(s => s.includes('COMPARE')),
      'has_question': possibleNext.find(s => s.includes('DIVE')),
      'ready_to_proceed': possibleNext.find(s => s.includes('ENROLL')),
      'wants_to_change': possibleNext.find(s => s.includes('INTRO'))
    };

    return intentToState[userIntent] || possibleNext[0];
  }
}

// Usage
const agent = new MetaPromptStateMachine();

const taskDescription = `
Help users research and compare health insurance plans.
They need to understand plan types, compare costs, 
check doctor networks, and complete enrollment.
`;

// Generate state machine from task description
const stateMachine = await agent.generateStateMachine(taskDescription);

console.log('Generated state machine:', JSON.stringify(stateMachine, null, 2));

// Run voice agent with generated state machine
await agent.runVoiceAgent(stateMachine, taskDescription);

function validateStateMachine(stateMachine) {
  // 1. Every state must have at least one next state or be terminal
  for (const [name, state] of Object.entries(stateMachine.states)) {
    if (!state.next && !state.terminal) {
      throw new Error(`State ${name} has no next states and is not terminal`);
    }
  }

  // 2. All referenced next states must exist
  for (const state of Object.values(stateMachine.states)) {
    if (state.next) {
      for (const nextState of state.next) {
        if (!stateMachine.states[nextState]) {
          throw new Error(`Referenced state ${nextState} does not exist`);
        }
      }
    }
  }

  // 3. Must have exactly one starting state (no incoming transitions)
  const statesWithIncoming = new Set();
  for (const state of Object.values(stateMachine.states)) {
    if (state.next) {
      state.next.forEach(s => statesWithIncoming.add(s));
    }
  }
  
  const startStates = Object.keys(stateMachine.states)
    .filter(name => !statesWithIncoming.has(name));
    
  if (startStates.length !== 1) {
    throw new Error(`Must have exactly one start state, found ${startStates.length}`);
  }

  // 4. Must have at least one terminal state
  const terminalStates = Object.values(stateMachine.states)
    .filter(s => s.terminal);
    
  if (terminalStates.length === 0) {
    throw new Error('No terminal state found');
  }

  return true;
}