Voice-First Tool Design Patterns: Redesigning APIs For Speech

// Tool designed for text agents
const searchProducts = {
  name: "search_products",
  description: "Search product catalog",
  parameters: {
    type: "object",
    properties: {
      query: {
        type: "string",
        description: "Search query"
      },
      filters: {
        type: "object",
        properties: {
          category: { type: "string" },
          minPrice: { type: "number" },
          maxPrice: { type: "number" },
          inStock: { type: "boolean" }
        }
      },
      sort: {
        type: "string",
        enum: ["price_asc", "price_desc", "rating", "newest"]
      },
      page: { type: "number", default: 1 },
      limit: { type: "number", default: 20 }
    },
    required: ["query"]
  }
};

// Voice-optimized tool
const findProduct = {
  name: "find_product",
  description: "Find products based on conversational request",
  parameters: {
    type: "object",
    properties: {
      request: {
        type: "string",
        description: "What the user is looking for, in their own words"
      },
      context: {
        type: "string",
        description: "Additional preferences or constraints mentioned"
      }
    },
    required: ["request"]
  }
};

// Tool implementation
async function findProduct({ request, context }) {
  // Parse natural language on the backend
  const intent = await parseUserIntent(request, context);
  
  return await searchCatalog({
    query: intent.product,
    filters: intent.filters,
    sort: intent.preferredSort
  });
}

// ❌ Bad: Low-level tools require multiple calls
const tools = [
  { name: "search_products" },
  { name: "filter_results" },
  { name: "sort_results" },
  { name: "get_product_details" }
];

// User says: "Find the best rated laptop under $800"
// Agent flow:
// 1. search_products(query="laptop")
// 2. filter_results(maxPrice=800)
// 3. sort_results(by="rating")
// 4. get_product_details(top result)
// Result: 4 tool calls, ~8 seconds, awkward pauses

// ✅ Good: High-level tool handles the workflow
const findBestMatch = {
  name: "find_best_match",
  description: "Find the best product matching user criteria",
  parameters: {
    type: "object",
    properties: {
      what: {
        type: "string",
        description: "What they're looking for"
      },
      constraints: {
        type: "string", 
        description: "Budget, ratings, features, etc."
      }
    }
  }
};

// User says same thing
// Agent flow:
// 1. find_best_match(what="laptop", constraints="best rated under $800")
// Result: 1 tool call, ~2 seconds, smooth conversation

const bookFlight = {
  name: "book_flight",
  description: "Book a flight. IMPORTANT: Tell the user you're checking availability before calling this.",
  parameters: {
    type: "object",
    properties: {
      route: { 
        type: "string",
        description: "Origin to destination" 
      },
      date: { type: "string" },
      passengers: { type: "number" }
    }
  },
  guidance: {
    beforeCall: "I'm checking flight availability",
    estimatedDuration: "3-5 seconds",
    duringExecution: "This might take a moment"
  }
};

# Text-first calendar tools
class CalendarTools:
    def list_events(self, start_date, end_date, calendar_id=None):
        """List calendar events in date range"""
        pass
    
    def get_event(self, event_id):
        """Get details of specific event"""
        pass
    
    def check_conflicts(self, proposed_start, proposed_end):
        """Check for scheduling conflicts"""
        pass
    
    def create_event(self, title, start, end, attendees, location):
        """Create new calendar event"""
        pass

# Voice agent flow to schedule a meeting:
# User: "Schedule a team meeting tomorrow at 2pm"
# Agent calls:
# 1. list_events(tomorrow_start, tomorrow_end)
# 2. check_conflicts(2pm, 3pm) 
# 3. create_event(...)
# Total: 3 calls, 5-7 seconds

# Voice-first calendar tool
class VoiceCalendarTool:
    def schedule_when_free(self, what, when, duration=None, who=None):
        """
        Schedule an event at the requested time if available.
        
        Args:
            what: What to schedule (e.g., "team meeting")
            when: When they want it (e.g., "tomorrow at 2pm")
            duration: How long (optional, defaults to 1 hour)
            who: Attendees mentioned (optional)
        
        Returns:
            Confirmation if successful, or suggests alternative if conflict
        
        Guidance for agent:
            - Say "Let me check your calendar" before calling
            - Estimated time: 2-3 seconds
            - If conflict found, suggest the next available slot
        """
        # Parse natural language
        parsed_time = parse_time_expression(when)
        
        # Check availability
        conflicts = self.check_conflicts(
            parsed_time.start,
            parsed_time.end
        )
        
        if conflicts:
            # Find next available
            next_slot = self.find_next_slot(
                after=parsed_time.start,
                duration=duration
            )
            return {
                "success": False,
                "message": f"You have a conflict at {when}. You're free at {next_slot}. Should I schedule it then?",
                "alternative": next_slot
            }
        
        # Book it
        event = self.create_event(
            title=what,
            start=parsed_time.start,
            end=parsed_time.end,
            attendees=parse_attendees(who) if who else []
        )
        
        return {
            "success": True,
            "message": f"Done. I've scheduled {what} for {when}.",
            "event_id": event.id
        }

# Voice agent flow with optimized tool:
# User: "Schedule a team meeting tomorrow at 2pm"
# Agent: "Let me check your calendar"
# Agent calls: schedule_when_free(what="team meeting", when="tomorrow at 2pm")
# Total: 1 call, 2 seconds

import { OpenAI } from 'openai';

const client = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY
});

// Voice-first tool definition
const resolveCustomerIssue = {
  type: "function",
  function: {
    name: "resolve_customer_issue",
    description: `Help resolve a customer's problem. 
    
    AGENT GUIDANCE:
    - Before calling, say: "Let me look into that for you"
    - This takes 3-5 seconds
    - If you find a solution, explain it clearly
    - If you need more info, ask the customer`,
    
    parameters: {
      type: "object",
      properties: {
        issue_description: {
          type: "string",
          description: "What the customer described in their own words"
        },
        account_context: {
          type: "string",
          description: "Any account details mentioned (order number, email, etc.)"
        },
        attempted_solutions: {
          type: "string",
          description: "What the customer already tried, if mentioned"
        }
      },
      required: ["issue_description"]
    }
  }
};

// Tool implementation
async function resolveCustomerIssue({
  issue_description,
  account_context,
  attempted_solutions
}) {
  // Use AI to parse the natural language
  const analysis = await client.chat.completions.create({
    model: "gpt-4",
    messages: [{
      role: "system",
      content: "Extract structured issue data from customer description"
    }, {
      role: "user",
      content: JSON.stringify({
        issue: issue_description,
        context: account_context,
        tried: attempted_solutions
      })
    }],
    response_format: { type: "json_object" }
  });
  
  const structured = JSON.parse(analysis.choices[0].message.content);
  
  // Look up the issue in knowledge base
  const solutions = await searchKnowledgeBase(structured);
  
  // Check customer's account
  const accountStatus = await checkAccount(structured.account_identifiers);
  
  // Apply any automated fixes
  const resolution = await attemptAutoResolution(structured, accountStatus);
  
  if (resolution.success) {
    return {
      resolved: true,
      solution: resolution.description,
      message_for_customer: `I've ${resolution.action}. ${resolution.next_steps}`,
      follow_up_needed: false
    };
  }
  
  return {
    resolved: false,
    suggestions: solutions.top_articles,
    message_for_customer: `Here's what usually works for this issue: ${solutions.summary}. Would you like to try that?`,
    follow_up_needed: true
  };
}

// Voice agent with optimized tool
const response = await client.chat.completions.create({
  model: "gpt-realtime",
  modalities: ["text", "audio"],
  tools: [resolveCustomerIssue],
  messages: [{
    role: "system",
    content: `You're a helpful customer support agent. 
    
    When a customer describes an issue:
    1. Listen completely
    2. Use resolve_customer_issue tool (it already knows to say "let me look into that")
    3. Explain the solution conversationally
    4. Ask if they need anything else`
  }, {
    role: "user",
    content: "My order still hasn't arrived and it's been two weeks"
  }]
});

// Agent naturally says:
// "Let me look into that for you..."
// [Calls tool]
// "I see your order #12345 is currently delayed. I've expedited the shipping 
//  and you should receive it by Thursday. I've also applied a $10 credit 
//  to your account. Is there anything else I can help with?"

const doWhatIMean = {
  name: "handle_user_request",
  description: "Flexible tool that interprets intent and takes appropriate action",
  parameters: {
    type: "object",
    properties: {
      user_said: { type: "string" },
      likely_intent: { 
        type: "string",
        enum: ["search", "book", "modify", "cancel", "info"]
      }
    }
  }
};

const updateWithContext = {
  name: "update_with_context",
  description: "Update something based on current conversation context",
  parameters: {
    type: "object",
    properties: {
      what_changed: { 
        type: "string",
        description: "What the user wants to change, in their words"
      },
      reference: {
        type: "string",
        description: "What they're referring to (might be implicit)"
      }
    }
  }
};

// User: "Actually, make it for 6 people instead"
// Tool receives: what_changed="6 people", reference="the reservation we just discussed"

const scheduleAppointment = {
  name: "schedule_appointment", 
  description: "Schedule appointment with intelligent defaults",
  parameters: {
    type: "object",
    properties: {
      when: {
        type: "string",
        description: "When they want to meet. Tool automatically suggests specific time if vague."
      },
      what: { type: "string" },
      duration: {
        type: "string",
        description: "Optional. Tool chooses appropriate default based on appointment type."
      }
    }
  }
};

// User: "Book a dentist appointment next week"
// Tool receives: when="next week", what="dentist appointment"
// Tool internally: suggests Tuesday 10am (first available), defaults to 1 hour (standard dental)

// Voice tool test suite
describe('Voice-optimized tools', () => {
  test('handles vague time expressions', async () => {
    const result = await scheduleAppointment({
      when: "sometime next week",
      what: "dentist"
    });
    
    expect(result.proposed_time).toBeDefined();
    expect(result.message).toContain("next week");
  });
  
  test('works with minimal parameters', async () => {
    const result = await findProduct({
      request: "laptop"
    });
    
    // Should still return results with sensible defaults
    expect(result.products.length).toBeGreaterThan(0);
  });
  
  test('returns conversational responses', async () => {
    const result = await bookFlight({
      route: "NYC to LA",
      date: "tomorrow"
    });
    
    // Response should be speakable
    expect(result.message).not.toContain('status=200');
    expect(result.message).toMatch(/booked|scheduled|confirmed/i);
  });
  
  test('handles ambiguity gracefully', async () => {
    const result = await scheduleAppointment({
      when: "afternoon",  // Vague
      what: "meeting"     // No attendees specified
    });
    
    // Should ask for clarification, not crash
    expect(result.needs_clarification).toBe(true);
    expect(result.questions).toBeDefined();
  });
});

// Wrap existing tool
function voiceWrapper(textTool) {
  return {
    name: `voice_${textTool.name}`,
    description: textTool.description + "\n\nAccepts natural language input.",
    parameters: {
      type: "object",
      properties: {
        natural_request: { 
          type: "string",
          description: "User's request in their own words"
        }
      }
    },
    async execute(params) {
      // Parse to structured format
      const structured = await parseNaturalLanguage(
        params.natural_request,
        textTool.parameters
      );
      
      // Call original tool
      return await textTool.execute(structured);
    }
  };
}