Voice Prompts Need Workflows, Not Vibes: State Machines for Structured Conversations

const CONVERSATION_STATES = {
  GREETING: {
    goal: "Welcome user and understand their intent",
    tools: [],
    exitConditions: ["intent_detected", "escalation_needed"],
    transitions: {
      intent_detected: "REQUIREMENTS",
      escalation_needed: "ESCALATE"
    }
  },
  
  REQUIREMENTS: {
    goal: "Gather all information needed to complete task",
    tools: ["ask_clarifying_question", "validate_input"],
    exitConditions: ["requirements_complete", "requirements_invalid"],
    transitions: {
      requirements_complete: "CONFIRMATION",
      requirements_invalid: "CLARIFICATION"
    }
  },
  
  CONFIRMATION: {
    goal: "Summarize what will happen and get explicit approval",
    tools: ["summarize_action"],
    exitConditions: ["user_confirms", "user_modifies", "user_cancels"],
    transitions: {
      user_confirms: "EXECUTION",
      user_modifies: "REQUIREMENTS",
      user_cancels: "CANCELLATION"
    }
  },
  
  EXECUTION: {
    goal: "Perform the requested action",
    tools: ["execute_action", "call_external_api"],
    exitConditions: ["action_success", "action_error"],
    transitions: {
      action_success: "SUMMARY",
      action_error: "ERROR_HANDLING"
    }
  },
  
  SUMMARY: {
    goal: "Tell user what was accomplished",
    tools: [],
    exitConditions: ["summary_delivered"],
    transitions: {
      summary_delivered: "CLOSING"
    }
  },
  
  CLOSING: {
    goal: "Offer help with anything else or end conversation",
    tools: [],
    exitConditions: ["conversation_continues", "conversation_ends"],
    transitions: {
      conversation_continues: "REQUIREMENTS",
      conversation_ends: "END"
    }
  },
  
  ERROR_HANDLING: {
    goal: "Explain error and offer recovery path",
    tools: ["diagnose_error", "suggest_alternative"],
    exitConditions: ["error_recoverable", "error_fatal"],
    transitions: {
      error_recoverable: "EXECUTION",
      error_fatal: "ESCALATE"
    }
  },
  
  ESCALATE: {
    goal: "Hand off to human with full context",
    tools: ["prepare_handoff_context", "notify_human_agent"],
    exitConditions: ["handoff_complete"],
    transitions: {
      handoff_complete: "END"
    }
  }
};

class StatefulVoiceAgent {
  constructor() {
    this.currentState = "GREETING";
    this.conversationContext = {
      userIntent: null,
      requirements: {},
      completedSteps: []
    };
  }
  
  generateMetaprompt() {
    const state = CONVERSATION_STATES[this.currentState];
    
    return `You are a voice assistant in the ${this.currentState} state.

CURRENT GOAL:
${state.goal}

AVAILABLE TOOLS:
${state.tools.length > 0 ? state.tools.join(', ') : 'None - use conversation only'}

EXIT CONDITIONS (watch for these):
${state.exitConditions.map(c => `- ${c}`).join('\n')}

CONTEXT FROM CONVERSATION:
${JSON.stringify(this.conversationContext, null, 2)}

INSTRUCTIONS FOR THIS STATE:
${this.getStateInstructions()}

When an exit condition is met, indicate which one so we can transition states.
`;
  }
  
  getStateInstructions() {
    const instructions = {
      GREETING: `1. Greet the user warmly
2. Ask how you can help
3. Listen for their intent
4. If intent is clear, confirm understanding
5. If intent is unclear, ask ONE clarifying question
6. If request is outside your capabilities, prepare to escalate`,
      
      REQUIREMENTS: `1. Ask focused questions to gather needed information
2. Validate each input as you receive it
3. Confirm you've understood correctly
4. Don't move forward until ALL requirements are gathered
5. If user provides invalid input, explain why and re-ask
6. Track what you still need vs what you have`,
      
      CONFIRMATION: `1. Summarize what you're about to do
2. Be specific: names, amounts, dates, actions
3. Ask explicit confirmation: "Should I proceed?"
4. Don't execute without explicit "yes"
5. If user wants changes, return to requirements gathering`,
      
      EXECUTION: `1. Tell user you're performing the action
2. Use available tools to execute
3. Monitor for errors
4. Don't hallucinate success - use tool results
5. If error occurs, explain clearly what went wrong`,
      
      SUMMARY: `1. Confirm what was accomplished
2. Provide any relevant reference numbers or details
3. Keep it brief - they already know what happened
4. Set up for closing`,
      
      CLOSING: `1. Ask if there's anything else you can help with
2. If yes, restart the requirements gathering process
3. If no, end warmly and professionally
4. Don't drag out the goodbye`,
      
      ERROR_HANDLING: `1. Apologize for the issue
2. Explain what went wrong in simple terms
3. Offer a recovery path if possible
4. If not recoverable, offer to escalate to human
5. Don't try to hide or minimize errors`,
      
      ESCALATE: `1. Explain you're connecting them to a specialist
2. Assure them the specialist will have full context
3. Summarize what you've covered so far
4. Hand off gracefully`
    };
    
    return instructions[this.currentState] || "Follow conversation naturally.";
  }
  
  async transition(exitCondition) {
    const currentStateConfig = CONVERSATION_STATES[this.currentState];
    const nextState = currentStateConfig.transitions[exitCondition];
    
    if (!nextState) {
      console.error(`No transition defined for ${exitCondition} from ${this.currentState}`);
      return;
    }
    
    console.log(`[State] Transitioning: ${this.currentState} -> ${nextState} (${exitCondition})`);
    
    this.currentState = nextState;
    this.conversationContext.completedSteps.push({
      state: this.currentState,
      timestamp: Date.now(),
      trigger: exitCondition
    });
  }
  
  async processUserInput(transcript) {
    // Generate state-specific metaprompt
    const metaprompt = this.generateMetaprompt();
    
    // Send to OpenAI with current state context
    const response = await openai.chat.completions.create({
      model: "gpt-realtime",
      messages: [
        { role: "system", content: metaprompt },
        { role: "user", content: transcript }
      ],
      tools: this.getAvailableTools()
    });
    
    // Check if agent indicated an exit condition
    const detectedExit = this.detectExitCondition(response);
    if (detectedExit) {
      await this.transition(detectedExit);
    }
    
    return response.choices[0].message.content;
  }
  
  detectExitCondition(response) {
    const stateConfig = CONVERSATION_STATES[this.currentState];
    const content = response.choices[0].message.content.toLowerCase();
    
    // Simple pattern matching - in production, use more robust detection
    for (let condition of stateConfig.exitConditions) {
      if (content.includes(condition.replace(/_/g, ' '))) {
        return condition;
      }
    }
    
    return null;
  }
  
  getAvailableTools() {
    const stateConfig = CONVERSATION_STATES[this.currentState];
    return stateConfig.tools.map(toolName => TOOL_DEFINITIONS[toolName]);
  }
}

from enum import Enum
from typing import Dict, List, Optional
from dataclasses import dataclass

class ConversationState(Enum):
    GREETING = "greeting"
    REQUIREMENTS = "requirements"
    CONFIRMATION = "confirmation"
    EXECUTION = "execution"
    SUMMARY = "summary"
    CLOSING = "closing"
    ERROR_HANDLING = "error_handling"
    ESCALATE = "escalate"
    END = "end"

@dataclass
class StateConfig:
    goal: str
    tools: List[str]
    exit_conditions: List[str]
    transitions: Dict[str, ConversationState]
    instructions: str

class StatefulVoiceAgent:
    def __init__(self):
        self.current_state = ConversationState.GREETING
        self.conversation_context = {
            "user_intent": None,
            "requirements": {},
            "completed_steps": []
        }
        
        self.state_configs = self._define_states()
    
    def _define_states(self) -> Dict[ConversationState, StateConfig]:
        return {
            ConversationState.GREETING: StateConfig(
                goal="Welcome user and understand their intent",
                tools=[],
                exit_conditions=["intent_detected", "escalation_needed"],
                transitions={
                    "intent_detected": ConversationState.REQUIREMENTS,
                    "escalation_needed": ConversationState.ESCALATE
                },
                instructions="""1. Greet warmly
2. Ask how you can help
3. Listen for intent
4. Confirm understanding
5. One clarifying question if needed"""
            ),
            
            ConversationState.REQUIREMENTS: StateConfig(
                goal="Gather all information needed",
                tools=["ask_clarifying_question", "validate_input"],
                exit_conditions=["requirements_complete", "requirements_invalid"],
                transitions={
                    "requirements_complete": ConversationState.CONFIRMATION,
                    "requirements_invalid": ConversationState.GREETING
                },
                instructions="""1. Ask focused questions
2. Validate each input
3. Confirm understanding
4. Don't proceed until complete
5. Track what you still need"""
            ),
            
            ConversationState.CONFIRMATION: StateConfig(
                goal="Summarize and get approval",
                tools=["summarize_action"],
                exit_conditions=["user_confirms", "user_modifies", "user_cancels"],
                transitions={
                    "user_confirms": ConversationState.EXECUTION,
                    "user_modifies": ConversationState.REQUIREMENTS,
                    "user_cancels": ConversationState.CLOSING
                },
                instructions="""1. Summarize specifically
2. Ask explicit confirmation
3. Don't execute without 'yes'
4. Handle changes gracefully"""
            ),
            
            ConversationState.EXECUTION: StateConfig(
                goal="Perform the action",
                tools=["execute_action", "call_api"],
                exit_conditions=["action_success", "action_error"],
                transitions={
                    "action_success": ConversationState.SUMMARY,
                    "action_error": ConversationState.ERROR_HANDLING
                },
                instructions="""1. Announce you're executing
2. Use tools to perform action
3. Monitor for errors
4. Don't hallucinate results"""
            ),
            
            ConversationState.SUMMARY: StateConfig(
                goal="Confirm what was done",
                tools=[],
                exit_conditions=["summary_delivered"],
                transitions={
                    "summary_delivered": ConversationState.CLOSING
                },
                instructions="""1. Confirm completion
2. Provide reference details
3. Keep it brief
4. Set up for closing"""
            ),
            
            ConversationState.CLOSING: StateConfig(
                goal="Offer more help or end",
                tools=[],
                exit_conditions=["conversation_continues", "conversation_ends"],
                transitions={
                    "conversation_continues": ConversationState.REQUIREMENTS,
                    "conversation_ends": ConversationState.END
                },
                instructions="""1. Ask about anything else
2. If yes, restart flow
3. If no, end professionally"""
            )
        }
    
    def generate_metaprompt(self) -> str:
        """Generate state-specific prompt."""
        config = self.state_configs[self.current_state]
        
        return f"""You are in the {self.current_state.value.upper()} state.

GOAL: {config.goal}

AVAILABLE TOOLS: {', '.join(config.tools) if config.tools else 'None'}

EXIT CONDITIONS:
{chr(10).join(f'- {cond}' for cond in config.exit_conditions)}

INSTRUCTIONS:
{config.instructions}

CONTEXT:
{self.conversation_context}

When an exit condition is met, clearly indicate which one.
"""
    
    def transition(self, exit_condition: str):
        """Transition to next state based on exit condition."""
        config = self.state_configs[self.current_state]
        
        if exit_condition not in config.transitions:
            print(f"Warning: No transition for {exit_condition} from {self.current_state}")
            return
        
        next_state = config.transitions[exit_condition]
        
        print(f"[State] {self.current_state.value} -> {next_state.value} ({exit_condition})")
        
        self.current_state = next_state
        self.conversation_context["completed_steps"].append({
            "state": self.current_state.value,
            "trigger": exit_condition,
            "timestamp": "now"
        })
    
    async def process_input(self, transcript: str) -> str:
        """Process user input with state-aware prompt."""
        metaprompt = self.generate_metaprompt()
        
        # Send to OpenAI with metaprompt
        response = await self.call_openai(metaprompt, transcript)
        
        # Detect exit condition and transition
        exit_condition = self.detect_exit_condition(response)
        if exit_condition:
            self.transition(exit_condition)
        
        return response
    
    def detect_exit_condition(self, response: str) -> Optional[str]:
        """Detect which exit condition was triggered."""
        config = self.state_configs[self.current_state]
        response_lower = response.lower()
        
        for condition in config.exit_conditions:
            if condition.replace('_', ' ') in response_lower:
                return condition
        
        return None

# Usage
agent = StatefulVoiceAgent()

# Conversation flow
await agent.process_input("I need to create a workspace")
# State: GREETING -> REQUIREMENTS

await agent.process_input("Name it Q1 Campaign, shared with team")
# State: REQUIREMENTS -> CONFIRMATION

await agent.process_input("Yes, create it")
# State: CONFIRMATION -> EXECUTION -> SUMMARY -> CLOSING

REQUIREMENTS: {
  substates: {
    GATHERING: "Asking questions",
    VALIDATING: "Checking inputs",
    CONFIRMING_REQUIREMENTS: "Ensuring completeness"
  }
}

transitions: {
  requirements_complete: (context) => {
    return context.requirements.complex 
      ? "SPECIALIST_REVIEW" 
      : "CONFIRMATION";
  }
}

primaryFlow: GREETING -> REQUIREMENTS -> EXECUTION
monitoringFlow: LISTENING_FOR_INTERRUPTS (always active)