Agentic-AI/src/speech/vad.js

// ============================================
// Endpointing — Silero VAD via onnxruntime-node.
//
// Deciding turn boundaries on the server rather than in the browser keeps the
// rule in one place for every future channel (a phone bridge has no
// AudioWorklet), and gives the server the signal it needs for barge-in: it has
// to know the user started talking while the assistant was still speaking.
// ============================================
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import fs from 'node:fs/promises';
import ort from 'onnxruntime-node';
import logger from '../utils/logger.js';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
const MODEL_URL = 'https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx';
const MODEL_PATH = path.join(process.env.SPEECH_CACHE_DIR || path.join(ROOT, '.transformers-cache'), 'silero-vad.onnx');

// Silero wants exactly 512 samples at 16 kHz (32 ms). The browser sends 40 ms
// chunks, so audio is buffered and drained in exact frames rather than forcing
// the client to match.
const FRAME = 512;
const RATE = 16000;
const FRAME_MS = (FRAME / RATE) * 1000;

let sessionPromise = null;

async function getSession() {
  if (sessionPromise) return sessionPromise;
  sessionPromise = (async () => {
    try {
      await fs.access(MODEL_PATH);
    } catch {
      logger.info('⬇️  fetching Silero VAD (2 MB)…');
      const res = await fetch(MODEL_URL);
      if (!res.ok) throw new Error(`VAD download failed: ${res.status}`);
      await fs.mkdir(path.dirname(MODEL_PATH), { recursive: true });
      await fs.writeFile(MODEL_PATH, Buffer.from(await res.arrayBuffer()));
    }
    const s = await ort.InferenceSession.create(MODEL_PATH);
    logger.info('🎚️  Silero VAD ready');
    return s;
  })().catch((e) => { sessionPromise = null; throw e; });
  return sessionPromise;
}

export const vadOptions = {
  threshold: Number(process.env.VAD_THRESHOLD ?? 0.5),
  // Trailing silence that ends a turn. Too short truncates someone who pauses
  // mid-sentence; too long makes the assistant feel sluggish.
  silenceMs: Number(process.env.VAD_SILENCE_MS ?? 700),
  // Ignore blips, so a cough or a door does not open a turn.
  minSpeechMs: Number(process.env.VAD_MIN_SPEECH_MS ?? 250),
  // Audio kept from BEFORE detection, so word onsets are not clipped.
  prefixMs: Number(process.env.VAD_PREFIX_MS ?? 300),
  maxUtteranceMs: Number(process.env.VAD_MAX_UTTERANCE_MS ?? 30000),
};

/** Streaming endpointer. One instance per connection. */
export class Endpointer {
  constructor(opts = {}) {
    this.o = { ...vadOptions, ...opts };
    this.pending = new Float32Array(0);
    this.prefixFrames = Math.max(1, Math.round(this.o.prefixMs / FRAME_MS));
    this.reset();
  }

  reset() {
    this.speaking = false;
    this.speechMs = 0;
    this.silenceMs = 0;
    this.buffer = [];
    this.prefix = [];
    // Silero is recurrent: this 2x1x128 state carries across frames and must
    // be reset between turns or the model stays biased by the last utterance.
    this.state = new ort.Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128]);
    this.pending = new Float32Array(0);
  }

  /**
   * Feed float32 mono @16k.
   * @returns {Promise<{utterances: Float32Array[], started: boolean}>}
   *   `started` flips the moment speech begins — that is the barge-in signal.
   */
  async push(pcm) {
    const session = await getSession();

    const merged = new Float32Array(this.pending.length + pcm.length);
    merged.set(this.pending);
    merged.set(pcm, this.pending.length);
    this.pending = merged;

    const utterances = [];
    let started = false;
    let offset = 0;

    while (this.pending.length - offset >= FRAME) {
      const frame = this.pending.subarray(offset, offset + FRAME);
      offset += FRAME;

      const out = await session.run({
        input: new ort.Tensor('float32', frame, [1, FRAME]),
        sr: new ort.Tensor('int64', BigInt64Array.from([BigInt(RATE)]), []),
        state: this.state,
      });
      this.state = out.stateN ?? out.state_n ?? this.state;
      const voiced = out.output.data[0] >= this.o.threshold;

      if (!this.speaking) {
        this.prefix.push(Float32Array.from(frame));
        if (this.prefix.length > this.prefixFrames) this.prefix.shift();

        if (voiced) {
          this.speechMs += FRAME_MS;
          if (this.speechMs >= this.o.minSpeechMs) {
            this.speaking = true;
            this.silenceMs = 0;
            this.buffer = this.prefix;          // open the turn with the pre-roll
            this.prefix = [];
            started = true;
          }
        } else {
          this.speechMs = 0;
        }
        continue;
      }

      this.buffer.push(Float32Array.from(frame));
      if (voiced) this.silenceMs = 0;
      else this.silenceMs += FRAME_MS;

      const spokenMs = this.buffer.length * FRAME_MS;
      if (this.silenceMs >= this.o.silenceMs || spokenMs >= this.o.maxUtteranceMs) {
        utterances.push(concat(this.buffer));
        this.reset();
      }
    }

    this.pending = this.pending.slice(offset);
    return { utterances, started };
  }
}

function concat(frames) {
  const total = frames.reduce((n, f) => n + f.length, 0);
  const out = new Float32Array(total);
  let i = 0;
  for (const f of frames) { out.set(f, i); i += f.length; }
  return out;
}

export const warmupVad = () => getSession().catch(() => {});