Voice As Assistive Technology For Motor Accessibility

// Voice-enabled (bad for accessibility):
// "Click the export button"
// → User must still click a button
//
// "Type in the search box"
// → User must still type
//
// "Select the third item"
// → User must still make a selection

// Voice-first (accessible):
// "Export my data as CSV"
// → No clicking required, agent handles entire flow
//
// "Find documents about Q4 revenue"
// → No typing required, agent performs search
//
// "Open the project plan from last Tuesday"
// → No selecting required, agent identifies and opens file

// BAD: Voice-enabled (requires fallback to mouse/keyboard)
const voiceEnabledCommands = {
  "click save": () => {
    highlightElement("#save-button");
    speakMessage("Save button is highlighted. Click it to save.");
    // User still needs to click
  }
};

// GOOD: Voice-first (completes action)
const voiceFirstCommands = {
  "save my work": async () => {
    const result = await saveDocument();
    if (result.success) {
      speakMessage(`Saved ${result.filename}. All changes are saved.`);
    } else {
      speakMessage(`Couldn't save. ${result.error}. Would you like me to try again?`);
    }
    // Entire action completed by voice
  }
};

// Traditional (requires 5 clicks):
// Settings → Account → Security → Password → Change
// User must navigate 5 layers, clicking each one

// Voice-first (single command):
agent.onCommand("change my password", async () => {
  // Skip directly to password change
  const result = await openPasswordChangeDialog();
  speakMessage("I've opened password change. Tell me your new password when ready.");
});

class VoiceNavigator {
  constructor() {
    this.commands = new Map();
  }
  
  register(phrase, action, context = {}) {
    this.commands.set(phrase, { action, context });
  }
  
  async execute(userPhrase) {
    // Find matching command (fuzzy match)
    const match = this.findBestMatch(userPhrase);
    
    if (match) {
      await match.action(match.context);
    } else {
      // Suggest nearby commands
      const suggestions = this.getSuggestions(userPhrase);
      await this.agent.say(`I'm not sure what you mean. Did you want to: ${suggestions.join(', ')}?`);
    }
  }
}

// Register direct access to everything
const nav = new VoiceNavigator();

// Deep features accessible directly
nav.register("change my password", async () => {
  await openPasswordChange();
});

nav.register("export last month's data", async () => {
  await exportData({ range: "last_month" });
});

nav.register("show me projects I'm assigned to", async () => {
  await filterProjects({ assigned_to: currentUser });
});

// No navigation required - voice goes straight to the feature

// BAD: Visual-only feedback
async function saveDocument() {
  // Show spinner on button
  button.classList.add('loading');
  
  await fetch('/api/save', { ... });
  
  // Show checkmark
  button.classList.add('success');
  
  // User has no idea what happened if they can't see screen
}

// GOOD: Spoken feedback
async function saveDocument() {
  await agent.say("Saving now.");
  
  try {
    const result = await fetch('/api/save', { ... });
    await agent.say(`Saved successfully. File is ${result.filename}.`);
  } catch (error) {
    await agent.say(`Save failed. ${error.message}. Would you like me to try again?`);
  }
}

async function processLargeFile(fileId) {
  await agent.say("Starting file processing. This usually takes 2 to 3 minutes.");
  
  const processor = startProcessing(fileId);
  
  // Narrate progress every 30 seconds
  const progressInterval = setInterval(async () => {
    const progress = processor.getProgress();
    await agent.say(`Still processing. ${progress.percentComplete}% complete. ${progress.estimatedSecondsRemaining} seconds remaining.`);
  }, 30000);
  
  try {
    const result = await processor.complete();
    clearInterval(progressInterval);
    await agent.say(`Processing complete. ${result.rowsProcessed} rows processed successfully.`);
  } catch (error) {
    clearInterval(progressInterval);
    await agent.say(`Processing failed: ${error.message}`);
  }
}

// Error shows visual message, requires mouse to dismiss
if (error) {
  showErrorDialog("An error occurred. Click OK to continue.");
  // User can't click OK
}

if (error) {
  await agent.say(`I encountered an error: ${error.message}. Would you like me to try again, or should I skip this step?`);
  
  const response = await agent.listenForResponse();
  
  if (response.includes("try again")) {
    await retryOperation();
  } else if (response.includes("skip")) {
    await skipToNextStep();
  } else {
    await agent.say("I can retry the operation or skip this step. Which would you like?");
  }
}

class VoiceActionHistory {
  constructor() {
    this.history = [];
    this.agent = null;
  }
  
  async execute(action, undoAction, description) {
    // Execute action
    const result = await action();
    
    // Record for undo
    this.history.push({
      description,
      undo: undoAction,
      timestamp: Date.now()
    });
    
    // Announce completion + undo option
    await this.agent.say(`${description}. Say 'undo' if you want to reverse this.`);
    
    return result;
  }
  
  async undo() {
    if (this.history.length === 0) {
      await this.agent.say("Nothing to undo.");
      return;
    }
    
    const lastAction = this.history.pop();
    await lastAction.undo();
    await this.agent.say(`Undid: ${lastAction.description}.`);
  }
}

// Usage:
const actionHistory = new VoiceActionHistory();

// Deletions can be undone
await actionHistory.execute(
  () => deleteFile(fileId),
  () => restoreFile(fileId),
  "Deleted project plan"
);

// User can immediately say "undo" if it was a mistake

class VoiceDocumentEditor {
  constructor(agent) {
    this.agent = agent;
    this.document = [];
    this.cursor = 0;
  }
  
  async handleCommand(command) {
    // Dictation
    if (command.type === 'dictation') {
      this.document.splice(this.cursor, 0, command.text);
      this.cursor += command.text.length;
      await this.agent.say(`Added: "${command.text}"`);
    }
    
    // Navigation
    else if (command.match(/go to (start|end|paragraph \d+)/)) {
      const location = this.parseLocation(command);
      this.cursor = location;
      await this.agent.say(`Moved to ${command.match(/go to (.*)/)[1]}`);
    }
    
    // Selection and editing
    else if (command.startsWith('select')) {
      const selection = this.parseSelection(command);
      await this.agent.say(`Selected: "${selection.text}"`);
      
      // User can now say "delete" or "bold" or "replace with..."
      this.activeSelection = selection;
    }
    
    else if (command.startsWith('delete') && this.activeSelection) {
      this.document.splice(this.activeSelection.start, this.activeSelection.length);
      await this.agent.say(`Deleted: "${this.activeSelection.text}"`);
      this.activeSelection = null;
    }
    
    else if (command.startsWith('bold') && this.activeSelection) {
      this.applyFormat(this.activeSelection, 'bold');
      await this.agent.say(`Made "${this.activeSelection.text}" bold`);
      this.activeSelection = null;
    }
    
    // Undo
    else if (command === 'undo') {
      await this.undo();
    }
    
    // Save
    else if (command.includes('save')) {
      await this.save();
      await this.agent.say("Document saved.");
    }
  }
}

// Voice-accessible menu
class VoiceMenu {
  constructor(items) {
    this.items = items;
  }
  
  async show() {
    const itemNames = this.items.map(i => i.name).join(', ');
    await agent.say(`Options are: ${itemNames}. Which would you like?`);
    
    const response = await agent.listen();
    
    const selected = this.items.find(i => 
      response.toLowerCase().includes(i.name.toLowerCase())
    );
    
    if (selected) {
      await selected.action();
    } else {
      await agent.say(`I didn't catch that. Say one of: ${itemNames}`);
      await this.show(); // Try again
    }
  }
}

// Voice-accessible form
class VoiceForm {
  constructor(fields) {
    this.fields = fields;
    this.values = {};
  }
  
  async fill() {
    for (const field of this.fields) {
      await agent.say(`${field.label}. ${field.prompt}`);
      
      const response = await agent.listen();
      this.values[field.name] = this.validate(response, field);
      
      if (field.confirm) {
        await agent.say(`You said: ${this.values[field.name]}. Is that correct?`);
        const confirmation = await agent.listen();
        
        if (!confirmation.includes('yes')) {
          // Re-ask this field
          this.fields.splice(this.fields.indexOf(field), 0, field);
        }
      }
    }
    
    return this.values;
  }
}

// Usage:
const form = new VoiceForm([
  {
    name: 'name',
    label: 'Full name',
    prompt: 'What is your full name?',
    confirm: true
  },
  {
    name: 'email',
    label: 'Email address',
    prompt: 'What is your email address?',
    confirm: true
  }
]);

const data = await form.fill();
// All form interaction through voice

// Flexible command matching
class FlexibleCommandMatcher {
  constructor() {
    this.commands = [];
  }
  
  register(patterns, action, description) {
    this.commands.push({ patterns, action, description });
  }
  
  async match(userPhrase) {
    for (const cmd of this.commands) {
      for (const pattern of cmd.patterns) {
        if (this.fuzzyMatch(userPhrase, pattern)) {
          return cmd;
        }
      }
    }
    return null;
  }
  
  fuzzyMatch(input, pattern) {
    // Match if input contains key words from pattern
    const patternWords = pattern.toLowerCase().split(' ');
    const inputLower = input.toLowerCase();
    
    const matchedWords = patternWords.filter(word => inputLower.includes(word));
    return matchedWords.length >= patternWords.length * 0.6; // 60% match threshold
  }
}

// Register multiple phrasings for same action
matcher.register(
  [
    "save my work",
    "save document",
    "save changes",
    "save file"
  ],
  () => saveDocument(),
  "Saves current document"
);

// User can say any variation