WebSocket Voice Protocol

Client                                Server
  |                                      |
  |  --- WebSocket CONNECT /audio --->   |
  |                                      |  (assign clientId, sessionId)
  |  <--- session_ack (JSON) ----------  |
  |                                      |
  |  --- session_start (JSON) -------->  |  (optional: set deviceId)
  |                                      |
  |  --- binary audio frames --------->  |  (microphone PCM data)
  |  --- binary audio frames --------->  |
  |                                      |  (STT processing)
  |                                      |  (LLM reasoning)
  |                                      |  (TTS synthesis)
  |  <--- tts_start (JSON) ----------   |
  |  <--- binary audio frames --------  |  (TTS PCM data)
  |  <--- binary audio frames --------  |
  |  <--- tts_end (JSON) ------------   |
  |                                      |
  |  --- session_end (JSON) --------->  |  (optional: end session)
  |                                      |
  |  --- WebSocket CLOSE ------------>  |
  |                                      |

const ws = new WebSocket('ws://localhost:3000/audio');

ws.binaryType = 'arraybuffer';

ws.onopen = () => {
  // Send session start with device identification
  ws.send(JSON.stringify({
    type: 'session_start',
    deviceId: 'browser-main'
  }));
};

ws.onmessage = (event) => {
  if (event.data instanceof ArrayBuffer) {
    // Binary frame: TTS audio data (Int16 PCM, 16kHz, mono)
    const pcmData = new Int16Array(event.data);
    playAudio(pcmData);
  } else {
    // Text frame: control message
    const msg = JSON.parse(event.data);
    switch (msg.type) {
      case 'session_ack':
        console.log('Session:', msg.sessionId);
        break;
      case 'tts_start':
        console.log('TTS playback starting');
        break;
      case 'tts_end':
        console.log('TTS playback complete');
        break;
      case 'error':
        console.error('Server error:', msg.error);
        break;
    }
  }
};

// Stream microphone audio
async function startMicrophone() {
  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
  const context = new AudioContext({ sampleRate: 16000 });
  const source = context.createMediaStreamSource(stream);
  const processor = context.createScriptProcessor(4096, 1, 1);

  source.connect(processor);
  processor.connect(context.destination);

  processor.onaudioprocess = (e) => {
    const float32 = e.inputBuffer.getChannelData(0);
    // Convert Float32 to Int16
    const int16 = new Int16Array(float32.length);
    for (let i = 0; i < float32.length; i++) {
      int16[i] = Math.max(-32768, Math.min(32767, float32[i] * 32768));
    }
    if (ws.readyState === WebSocket.OPEN) {
      ws.send(int16.buffer);
    }
  };
}