TypeScript Agents SDK For Voice Applications

// TypeScript SDK - Browser (WebRTC)
import { RealtimeClient } from "@openai/realtime-api-beta";

const client = new RealtimeClient({
  apiKey: process.env.OPENAI_API_KEY,
  // Automatically uses WebRTC in browser for ultra-low latency
});

// TypeScript SDK - Node.js (WebSocket)
const client = new RealtimeClient({
  apiKey: process.env.OPENAI_API_KEY,
  // Automatically uses WebSocket on server
});

# Python SDK - Server only (WebSocket)
from openai import realtime

client = realtime.RealtimeClient(
    api_key=os.environ["OPENAI_API_KEY"]
    # Always WebSocket (no browser environment)
)

from openai import agents, realtime
import os

# Define agent
agent = agents.Agent(
    name="booking_agent",
    model="gpt-realtime",
    instructions="""You are a restaurant booking agent.
    Your job: Help users book tables.
    Always confirm: party size, date, time, and name before booking.""",
    tools=[
        {
            "type": "function",
            "function": {
                "name": "book_table",
                "description": "Books a restaurant table",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "party_size": {"type": "number"},
                        "date": {"type": "string"},
                        "time": {"type": "string"},
                        "name": {"type": "string"}
                    },
                    "required": ["party_size", "date", "time", "name"]
                }
            }
        }
    ]
)

# Connect to Realtime API
async def run_agent():
    async with realtime.connect(agent) as session:
        # Voice interaction starts
        async for event in session.listen():
            if event.type == "tool_call":
                result = await book_table(**event.parameters)
                await session.send_tool_result(event.call_id, result)
            elif event.type == "conversation_complete":
                break

# Tool implementation
async def book_table(party_size, date, time, name):
    # Your booking logic here
    booking_id = create_booking(party_size, date, time, name)
    return {
        "success": True,
        "booking_id": booking_id,
        "message": f"Booked table for {party_size} on {date} at {time}"
    }

import { Agent, RealtimeClient } from "@openai/agents-sdk";

// Define agent (identical structure)
const agent = new Agent({
  name: "booking_agent",
  model: "gpt-realtime",
  instructions: `You are a restaurant booking agent.
    Your job: Help users book tables.
    Always confirm: party size, date, time, and name before booking.`,
  tools: [
    {
      type: "function",
      function: {
        name: "book_table",
        description: "Books a restaurant table",
        parameters: {
          type: "object",
          properties: {
            party_size: { type: "number" },
            date: { type: "string" },
            time: { type: "string" },
            name: { type: "string" }
          },
          required: ["party_size", "date", "time", "name"]
        }
      }
    }
  ]
});

// Connect to Realtime API
async function runAgent() {
  const session = await RealtimeClient.connect(agent);
  
  // Voice interaction starts
  for await (const event of session.listen()) {
    if (event.type === "tool_call") {
      const result = await bookTable(event.parameters);
      await session.sendToolResult(event.callId, result);
    } else if (event.type === "conversation_complete") {
      break;
    }
  }
}

// Tool implementation
async function bookTable(params: {
  party_size: number;
  date: string;
  time: string;
  name: string;
}) {
  // Your booking logic here
  const bookingId = createBooking(
    params.party_size,
    params.date,
    params.time,
    params.name
  );
  
  return {
    success: true,
    booking_id: bookingId,
    message: `Booked table for ${params.party_size} on ${params.date} at ${params.time}`
  };
}

# Python excels at backend tasks
async def process_large_dataset(file_path):
    df = pandas.read_csv(file_path)
    results = complex_analysis(df)
    return results

agent.add_tool(process_large_dataset)
# Heavy data processing on server

# Python has rich database ecosystem
async def query_customer_history(customer_id):
    async with db_pool.acquire() as conn:
        history = await conn.fetch(
            "SELECT * FROM orders WHERE customer_id = $1",
            customer_id
        )
    return history

agent.add_tool(query_customer_history)

# Python for ML inference
import torch
from transformers import pipeline

sentiment_analyzer = pipeline("sentiment-analysis")

async def analyze_sentiment(text):
    result = sentiment_analyzer(text)[0]
    return {
        "sentiment": result["label"],
        "confidence": result["score"]
    }

agent.add_tool(analyze_sentiment)

# Python for 24/7 server agents
async def main():
    while True:
        async with realtime.connect(agent) as session:
            await handle_session(session)
            # Automatically reconnects on disconnect

asyncio.run(main())

// TypeScript for client-side voice
import { RealtimeClient } from "@openai/realtime-api-beta";

// Runs entirely in browser
const client = new RealtimeClient({
  apiKey: getClientToken(), // Short-lived token from your backend
});

// WebRTC for lowest latency
await client.connect();
// No server required for voice connection

// TypeScript for immediate UI updates
session.on("agent_speaking", (event) => {
  // Update UI in real-time as agent speaks
  transcriptElement.textContent += event.text;
  
  // Show avatar animation
  avatarElement.classList.add("speaking");
});

session.on("agent_finished", () => {
  avatarElement.classList.remove("speaking");
});

// TypeScript for Vercel/Cloudflare Workers
export default async function handler(req: Request) {
  const agent = new Agent({ /* ... */ });
  const session = await RealtimeClient.connect(agent);
  
  // Runs on edge, closer to users
  const response = await session.handleRequest(req);
  return response;
}

// TypeScript for compile-time type checking
interface BookingParams {
  party_size: number;
  date: string; // ISO format
  time: string; // HH:MM format
  name: string;
}

async function bookTable(params: BookingParams): Promise<BookingResult> {
  // Compiler ensures you handle all fields correctly
  // Catches errors at build time, not runtime
}

// Frontend (TypeScript)
const agent = new Agent({
  name: "customer_service",
  model: "gpt-realtime",
  instructions: "You help customers with their accounts.",
  tools: [
    {
      type: "function",
      function: {
        name: "get_account_info",
        description: "Fetches customer account information"
      }
    }
  ]
});

// Tool calls backend
session.on("tool_call", async (event) => {
  if (event.name === "get_account_info") {
    // Call Python backend API
    const response = await fetch("/api/account", {
      method: "POST",
      body: JSON.stringify({ customer_id: event.parameters.customer_id })
    });
    
    const data = await response.json();
    await session.sendToolResult(event.callId, data);
  }
});

# Backend (Python)
from fastapi import FastAPI
import asyncpg

app = FastAPI()

@app.post("/api/account")
async def get_account_info(customer_id: str):
    # Python handles database queries
    async with db_pool.acquire() as conn:
        account = await conn.fetchrow(
            "SELECT * FROM accounts WHERE id = $1",
            customer_id
        )
        
    return {
        "account_id": account["id"],
        "balance": account["balance"],
        "status": account["status"],
        "history": await get_recent_transactions(customer_id)
    }

# Python agent
agent = agents.Agent(
    name="support",
    model="gpt-realtime",
    instructions="You are a support agent.",
    tools=[book_table_tool]
)

// TypeScript agent (nearly identical)
const agent = new Agent({
  name: "support",
  model: "gpt-realtime",
  instructions: "You are a support agent.",
  tools: [bookTableTool]
});