WeLe Agentic AI with Docker deployment

src/speech/vad.js

@@ -0,0 +1,152 @@
+// ============================================
+// Endpointing — Silero VAD via onnxruntime-node.
+//
+// Deciding turn boundaries on the server rather than in the browser keeps the
+// rule in one place for every future channel (a phone bridge has no
+// AudioWorklet), and gives the server the signal it needs for barge-in: it has
+// to know the user started talking while the assistant was still speaking.
+// ============================================
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import fs from 'node:fs/promises';
+import ort from 'onnxruntime-node';
+import logger from '../utils/logger.js';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+const MODEL_URL = 'https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx';
+const MODEL_PATH = path.join(process.env.SPEECH_CACHE_DIR || path.join(ROOT, '.transformers-cache'), 'silero-vad.onnx');
+
+// Silero wants exactly 512 samples at 16 kHz (32 ms). The browser sends 40 ms
+// chunks, so audio is buffered and drained in exact frames rather than forcing
+// the client to match.
+const FRAME = 512;
+const RATE = 16000;
+const FRAME_MS = (FRAME / RATE) * 1000;
+
+let sessionPromise = null;
+
+async function getSession() {
+  if (sessionPromise) return sessionPromise;
+  sessionPromise = (async () => {
+    try {
+      await fs.access(MODEL_PATH);
+    } catch {
+      logger.info('⬇️  fetching Silero VAD (2 MB)…');
+      const res = await fetch(MODEL_URL);
+      if (!res.ok) throw new Error(`VAD download failed: ${res.status}`);
+      await fs.mkdir(path.dirname(MODEL_PATH), { recursive: true });
+      await fs.writeFile(MODEL_PATH, Buffer.from(await res.arrayBuffer()));
+    }
+    const s = await ort.InferenceSession.create(MODEL_PATH);
+    logger.info('🎚️  Silero VAD ready');
+    return s;
+  })().catch((e) => { sessionPromise = null; throw e; });
+  return sessionPromise;
+}
+
+export const vadOptions = {
+  threshold: Number(process.env.VAD_THRESHOLD ?? 0.5),
+  // Trailing silence that ends a turn. Too short truncates someone who pauses
+  // mid-sentence; too long makes the assistant feel sluggish.
+  silenceMs: Number(process.env.VAD_SILENCE_MS ?? 700),
+  // Ignore blips, so a cough or a door does not open a turn.
+  minSpeechMs: Number(process.env.VAD_MIN_SPEECH_MS ?? 250),
+  // Audio kept from BEFORE detection, so word onsets are not clipped.
+  prefixMs: Number(process.env.VAD_PREFIX_MS ?? 300),
+  maxUtteranceMs: Number(process.env.VAD_MAX_UTTERANCE_MS ?? 30000),
+};
+
+/** Streaming endpointer. One instance per connection. */
+export class Endpointer {
+  constructor(opts = {}) {
+    this.o = { ...vadOptions, ...opts };
+    this.pending = new Float32Array(0);
+    this.prefixFrames = Math.max(1, Math.round(this.o.prefixMs / FRAME_MS));
+    this.reset();
+  }
+
+  reset() {
+    this.speaking = false;
+    this.speechMs = 0;
+    this.silenceMs = 0;
+    this.buffer = [];
+    this.prefix = [];
+    // Silero is recurrent: this 2x1x128 state carries across frames and must
+    // be reset between turns or the model stays biased by the last utterance.
+    this.state = new ort.Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128]);
+    this.pending = new Float32Array(0);
+  }
+
+  /**
+   * Feed float32 mono @16k.
+   * @returns {Promise<{utterances: Float32Array[], started: boolean}>}
+   *   `started` flips the moment speech begins — that is the barge-in signal.
+   */
+  async push(pcm) {
+    const session = await getSession();
+
+    const merged = new Float32Array(this.pending.length + pcm.length);
+    merged.set(this.pending);
+    merged.set(pcm, this.pending.length);
+    this.pending = merged;
+
+    const utterances = [];
+    let started = false;
+    let offset = 0;
+
+    while (this.pending.length - offset >= FRAME) {
+      const frame = this.pending.subarray(offset, offset + FRAME);
+      offset += FRAME;
+
+      const out = await session.run({
+        input: new ort.Tensor('float32', frame, [1, FRAME]),
+        sr: new ort.Tensor('int64', BigInt64Array.from([BigInt(RATE)]), []),
+        state: this.state,
+      });
+      this.state = out.stateN ?? out.state_n ?? this.state;
+      const voiced = out.output.data[0] >= this.o.threshold;
+
+      if (!this.speaking) {
+        this.prefix.push(Float32Array.from(frame));
+        if (this.prefix.length > this.prefixFrames) this.prefix.shift();
+
+        if (voiced) {
+          this.speechMs += FRAME_MS;
+          if (this.speechMs >= this.o.minSpeechMs) {
+            this.speaking = true;
+            this.silenceMs = 0;
+            this.buffer = this.prefix;          // open the turn with the pre-roll
+            this.prefix = [];
+            started = true;
+          }
+        } else {
+          this.speechMs = 0;
+        }
+        continue;
+      }
+
+      this.buffer.push(Float32Array.from(frame));
+      if (voiced) this.silenceMs = 0;
+      else this.silenceMs += FRAME_MS;
+
+      const spokenMs = this.buffer.length * FRAME_MS;
+      if (this.silenceMs >= this.o.silenceMs || spokenMs >= this.o.maxUtteranceMs) {
+        utterances.push(concat(this.buffer));
+        this.reset();
+      }
+    }
+
+    this.pending = this.pending.slice(offset);
+    return { utterances, started };
+  }
+}
+
+function concat(frames) {
+  const total = frames.reduce((n, f) => n + f.length, 0);
+  const out = new Float32Array(total);
+  let i = 0;
+  for (const f of frames) { out.set(f, i); i += f.length; }
+  return out;
+}
+
+export const warmupVad = () => getSession().catch(() => {});