// ============================================ // Endpointing — Silero VAD via onnxruntime-node. // // Deciding turn boundaries on the server rather than in the browser keeps the // rule in one place for every future channel (a phone bridge has no // AudioWorklet), and gives the server the signal it needs for barge-in: it has // to know the user started talking while the assistant was still speaking. // ============================================ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import fs from 'node:fs/promises'; import ort from 'onnxruntime-node'; import logger from '../utils/logger.js'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); const MODEL_URL = 'https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx'; const MODEL_PATH = path.join(process.env.SPEECH_CACHE_DIR || path.join(ROOT, '.transformers-cache'), 'silero-vad.onnx'); // Silero wants exactly 512 samples at 16 kHz (32 ms). The browser sends 40 ms // chunks, so audio is buffered and drained in exact frames rather than forcing // the client to match. const FRAME = 512; const RATE = 16000; const FRAME_MS = (FRAME / RATE) * 1000; let sessionPromise = null; async function getSession() { if (sessionPromise) return sessionPromise; sessionPromise = (async () => { try { await fs.access(MODEL_PATH); } catch { logger.info('⬇️ fetching Silero VAD (2 MB)…'); const res = await fetch(MODEL_URL); if (!res.ok) throw new Error(`VAD download failed: ${res.status}`); await fs.mkdir(path.dirname(MODEL_PATH), { recursive: true }); await fs.writeFile(MODEL_PATH, Buffer.from(await res.arrayBuffer())); } const s = await ort.InferenceSession.create(MODEL_PATH); logger.info('🎚️ Silero VAD ready'); return s; })().catch((e) => { sessionPromise = null; throw e; }); return sessionPromise; } export const vadOptions = { threshold: Number(process.env.VAD_THRESHOLD ?? 0.5), // Trailing silence that ends a turn. Too short truncates someone who pauses // mid-sentence; too long makes the assistant feel sluggish. silenceMs: Number(process.env.VAD_SILENCE_MS ?? 700), // Ignore blips, so a cough or a door does not open a turn. minSpeechMs: Number(process.env.VAD_MIN_SPEECH_MS ?? 250), // Audio kept from BEFORE detection, so word onsets are not clipped. prefixMs: Number(process.env.VAD_PREFIX_MS ?? 300), maxUtteranceMs: Number(process.env.VAD_MAX_UTTERANCE_MS ?? 30000), }; /** Streaming endpointer. One instance per connection. */ export class Endpointer { constructor(opts = {}) { this.o = { ...vadOptions, ...opts }; this.pending = new Float32Array(0); this.prefixFrames = Math.max(1, Math.round(this.o.prefixMs / FRAME_MS)); this.reset(); } reset() { this.speaking = false; this.speechMs = 0; this.silenceMs = 0; this.buffer = []; this.prefix = []; // Silero is recurrent: this 2x1x128 state carries across frames and must // be reset between turns or the model stays biased by the last utterance. this.state = new ort.Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128]); this.pending = new Float32Array(0); } /** * Feed float32 mono @16k. * @returns {Promise<{utterances: Float32Array[], started: boolean}>} * `started` flips the moment speech begins — that is the barge-in signal. */ async push(pcm) { const session = await getSession(); const merged = new Float32Array(this.pending.length + pcm.length); merged.set(this.pending); merged.set(pcm, this.pending.length); this.pending = merged; const utterances = []; let started = false; let offset = 0; while (this.pending.length - offset >= FRAME) { const frame = this.pending.subarray(offset, offset + FRAME); offset += FRAME; const out = await session.run({ input: new ort.Tensor('float32', frame, [1, FRAME]), sr: new ort.Tensor('int64', BigInt64Array.from([BigInt(RATE)]), []), state: this.state, }); this.state = out.stateN ?? out.state_n ?? this.state; const voiced = out.output.data[0] >= this.o.threshold; if (!this.speaking) { this.prefix.push(Float32Array.from(frame)); if (this.prefix.length > this.prefixFrames) this.prefix.shift(); if (voiced) { this.speechMs += FRAME_MS; if (this.speechMs >= this.o.minSpeechMs) { this.speaking = true; this.silenceMs = 0; this.buffer = this.prefix; // open the turn with the pre-roll this.prefix = []; started = true; } } else { this.speechMs = 0; } continue; } this.buffer.push(Float32Array.from(frame)); if (voiced) this.silenceMs = 0; else this.silenceMs += FRAME_MS; const spokenMs = this.buffer.length * FRAME_MS; if (this.silenceMs >= this.o.silenceMs || spokenMs >= this.o.maxUtteranceMs) { utterances.push(concat(this.buffer)); this.reset(); } } this.pending = this.pending.slice(offset); return { utterances, started }; } } function concat(frames) { const total = frames.reduce((n, f) => n + f.length, 0); const out = new Float32Array(total); let i = 0; for (const f of frames) { out.set(f, i); i += f.length; } return out; } export const warmupVad = () => getSession().catch(() => {});