diff --git a/.env.example b/.env.example index 6b02c94..48d0ee2 100644 --- a/.env.example +++ b/.env.example @@ -55,21 +55,3 @@ RATE_LIMIT_MAX=40 # --- Artifacts --- ARTIFACT_DIR=./storage/artifacts ARTIFACT_TTL_HOURS=72 - -# --- Voice (speech-to-speech, CPU, in-process) --- -# Models are ONNX via Transformers.js — no GPU, no Python, no second service. -# STT onnx-community/whisper-tiny.en English only; half the RAM of base -# TTS Xenova/mms-tts-eng from the Hub -# Tamil is built and tested (assets/tts/mms-tts-tam, exported locally — no -# public ONNX exists). Enable it with VOICE_LANGUAGES=en,ta and a multilingual -# STT_MODEL, but budget ~200 MB more resident for the extra voice. -VOICE_LANGUAGES=en -SPEECH_WARMUP=false -STT_MODEL=onnx-community/whisper-tiny.en -# Below this confidence, the user's preferred language beats the detector. -DETECT_CONFIDENCE=0.6 - -# Endpointing -VAD_SILENCE_MS=700 -VAD_MIN_SPEECH_MS=250 -VAD_PREFIX_MS=300 diff --git a/.gitignore b/.gitignore index 3027d65..d678a89 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,6 @@ __pycache__/ /huggingface/ /models/ .transformers-cache/ -*.onnx *.safetensors *.ckpt *.pt diff --git a/Dockerfile b/Dockerfile index 035d6db..7f86e27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,8 @@ # ============================================ # WeLe Agentic AI — production image # -# Speech runs in this same process: ONNX on CPU via Transformers.js. No GPU, -# no Python, no second service. -# -# node:20-slim, NOT alpine. onnxruntime-node ships glibc binaries and Alpine is -# musl, so the native module fails at load with: -# Error loading shared library ld-linux-x86-64.so.2 (needed by libonnxruntime.so.1) -# `sharp`, pulled in by Transformers.js, has the same constraint. +# Text-only agent service. node:20-slim rather than alpine: several native +# dependencies ship glibc binaries that will not load against musl. # ============================================ FROM node:20-slim AS deps @@ -28,10 +23,10 @@ COPY --from=deps /app/node_modules ./node_modules COPY package.json ./ COPY src/ ./src/ -# Both of these get a named volume mounted over them in compose. Docker seeds a -# NEW volume from the image path — including ownership — so they must exist here -# owned by `node`, or the unprivileged process gets EACCES on first write. -RUN mkdir -p /app/storage/artifacts /app/.transformers-cache && chown -R node:node /app/storage /app/.transformers-cache +# A named volume is mounted over this in compose. Docker seeds a NEW volume +# from the image path — including ownership — so it must exist here owned by +# `node`, or the unprivileged process gets EACCES on first write. +RUN mkdir -p /app/storage/artifacts && chown -R node:node /app/storage USER node EXPOSE 4000 diff --git a/assets/tts/mms-tts-tam/added_tokens.json b/assets/tts/mms-tts-tam/added_tokens.json deleted file mode 100644 index 76d0ecc..0000000 --- a/assets/tts/mms-tts-tam/added_tokens.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "": 58 -} diff --git a/assets/tts/mms-tts-tam/config.json b/assets/tts/mms-tts-tam/config.json deleted file mode 100644 index 5bd0d90..0000000 --- a/assets/tts/mms-tts-tam/config.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "activation_dropout": 0.1, - "architectures": [ - "VitsModel" - ], - "attention_dropout": 0.1, - "depth_separable_channels": 2, - "depth_separable_num_layers": 3, - "dtype": "float32", - "duration_predictor_dropout": 0.5, - "duration_predictor_filter_channels": 256, - "duration_predictor_flow_bins": 10, - "duration_predictor_kernel_size": 3, - "duration_predictor_num_flows": 4, - "duration_predictor_tail_bound": 5.0, - "ffn_dim": 768, - "ffn_kernel_size": 3, - "flow_size": 192, - "hidden_act": "relu", - "hidden_dropout": 0.1, - "hidden_size": 192, - "initializer_range": 0.02, - "layer_norm_eps": 1e-05, - "layerdrop": 0.1, - "leaky_relu_slope": 0.1, - "model_type": "vits", - "noise_scale": 0.667, - "noise_scale_duration": 0.8, - "num_attention_heads": 2, - "num_hidden_layers": 6, - "num_speakers": 1, - "posterior_encoder_num_wavenet_layers": 16, - "prior_encoder_num_flows": 4, - "prior_encoder_num_wavenet_layers": 4, - "resblock_dilation_sizes": [ - [ - 1, - 3, - 5 - ], - [ - 1, - 3, - 5 - ], - [ - 1, - 3, - 5 - ] - ], - "resblock_kernel_sizes": [ - 3, - 7, - 11 - ], - "sampling_rate": 16000, - "speaker_embedding_size": 0, - "speaking_rate": 1.0, - "spectrogram_bins": 513, - "transformers_version": "4.57.3", - "upsample_initial_channel": 512, - "upsample_kernel_sizes": [ - 16, - 16, - 4, - 4 - ], - "upsample_rates": [ - 8, - 8, - 2, - 2 - ], - "use_bias": true, - "use_stochastic_duration_prediction": true, - "vocab_size": 58, - "wavenet_dilation_rate": 1, - "wavenet_dropout": 0.0, - "wavenet_kernel_size": 5, - "window_size": 4 -} diff --git a/assets/tts/mms-tts-tam/quantize_config.json b/assets/tts/mms-tts-tam/quantize_config.json deleted file mode 100644 index c3b3fa4..0000000 --- a/assets/tts/mms-tts-tam/quantize_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "per_channel": false, - "reduce_range": false -} \ No newline at end of file diff --git a/assets/tts/mms-tts-tam/special_tokens_map.json b/assets/tts/mms-tts-tam/special_tokens_map.json deleted file mode 100644 index adc92cd..0000000 --- a/assets/tts/mms-tts-tam/special_tokens_map.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "pad_token": "3", - "unk_token": "" -} diff --git a/assets/tts/mms-tts-tam/tokenizer.json b/assets/tts/mms-tts-tam/tokenizer.json deleted file mode 100644 index c256cc2..0000000 --- a/assets/tts/mms-tts-tam/tokenizer.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "version": "1.0", - "truncation": null, - "padding": null, - "added_tokens": [ - { - "id": 58, - "content": "", - "single_word": false, - "lstrip": false, - "rstrip": false, - "normalized": false, - "special": true - } - ], - "normalizer": { - "type": "Sequence", - "normalizers": [ - { - "type": "Lowercase" - }, - { - "type": "Replace", - "pattern": { - "Regex": "[^012345679 '_aஅஆஇஈஉஊஎஏஐஒஓகஙசஜஞடணதநனபமயரறலளழவஷஸஹாிீுூெேைொோௌ்]" - }, - "content": "" - }, - { - "type": "Strip", - "strip_left": true, - "strip_right": true - }, - { - "type": "Replace", - "pattern": { - "Regex": "(?=.)|(?", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } - }, - "clean_up_tokenization_spaces": true, - "extra_special_tokens": {}, - "is_uroman": false, - "language": "tam", - "model_max_length": 1000000000000000019884624838656, - "normalize": true, - "pad_token": "3", - "phonemize": false, - "tokenizer_class": "VitsTokenizer", - "unk_token": "" -} diff --git a/assets/tts/mms-tts-tam/vocab.json b/assets/tts/mms-tts-tam/vocab.json deleted file mode 100644 index 086e82e..0000000 --- a/assets/tts/mms-tts-tam/vocab.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - " ": 7, - "'": 13, - "0": 47, - "1": 44, - "2": 23, - "3": 0, - "4": 54, - "5": 57, - "6": 36, - "7": 14, - "9": 31, - "_": 4, - "a": 15, - "அ": 1, - "ஆ": 45, - "இ": 38, - "ஈ": 2, - "உ": 3, - "ஊ": 11, - "எ": 37, - "ஏ": 16, - "ஐ": 52, - "ஒ": 27, - "ஓ": 49, - "க": 6, - "ங": 50, - "ச": 30, - "ஜ": 53, - "ஞ": 29, - "ட": 22, - "ண": 48, - "த": 41, - "ந": 5, - "ன": 35, - "ப": 46, - "ம": 26, - "ய": 39, - "ர": 25, - "ற": 28, - "ல": 21, - "ள": 43, - "ழ": 24, - "வ": 17, - "ஷ": 55, - "ஸ": 33, - "ஹ": 19, - "ா": 9, - "ி": 32, - "ீ": 12, - "ு": 51, - "ூ": 20, - "ெ": 10, - "ே": 8, - "ை": 34, - "ொ": 56, - "ோ": 42, - "ௌ": 40, - "்": 18 -} diff --git a/docker-compose.yml b/docker-compose.yml index 7b3bf03..80580f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,12 +28,6 @@ services: - NODE_ENV=production - PORT=4000 - # English-only keeps one TTS voice resident (~200 MB). Tamil is built and - # tested — VOICE_LANGUAGES=en,ta plus a multilingual STT_MODEL enables it, - # at roughly 200 MB more. - - VOICE_LANGUAGES=en - - STT_MODEL=onnx-community/whisper-tiny.en - # Reuse the CRM's Redis by service name on the shared network. # Keys are namespaced with REDIS_PREFIX, so the two never collide. - REDIS_ENABLED=true @@ -56,22 +50,13 @@ services: volumes: # Generated xlsx/pdf/pptx survive rebuilds; swept on a TTL by the app. - artifacts:/app/storage/artifacts - # Speech models are fetched from HuggingFace on first use (~200 MB). - # Without this they re-download on every restart and the first voice turn - # after a deploy stalls for a minute. - - speech_cache:/app/.transformers-cache # A 2 vCPU / 3.7 GB host already runs the CRM, chat-service, Redis and # Milvus. Capping this container keeps a runaway turn from starving them. - # - # 1 GB, not 768 MB: the speech models are resident once voice is used — - # measured 729 MB (Whisper tiny.en 415 MB + MMS-TTS English 203 MB + VAD - # 28 MB + the agent itself). 768 MB left no headroom, and an OOM kill takes - # text chat down with voice. Text-only sessions stay near 80 MB. deploy: resources: limits: - memory: 1024M + memory: 768M logging: driver: json-file @@ -90,4 +75,3 @@ networks: volumes: artifacts: - speech_cache: diff --git a/package-lock.json b/package-lock.json index 4a7c47b..a639336 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "0.1.0", "license": "ISC", "dependencies": { - "@huggingface/transformers": "^4.2.0", "@langchain/core": "^1.1.18", "@langchain/langgraph": "^1.1.0", "@langchain/openai": "^1.5.10", @@ -24,12 +23,10 @@ "ioredis": "^5.10.1", "jsonwebtoken": "^9.0.2", "mongoose": "^9.2.3", - "onnxruntime-node": "^1.24.3", "pdfkit": "^0.17.2", "pptxgenjs": "^4.0.1", "uuid": "^13.0.0", "winston": "^3.19.0", - "ws": "^8.21.3", "zod": "^3.25.76" } }, @@ -59,16 +56,6 @@ "kuler": "^2.0.0" } }, - "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@fast-csv/format": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", @@ -110,547 +97,6 @@ "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", "license": "MIT" }, - "node_modules/@huggingface/jinja": { - "version": "0.5.9", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.9.tgz", - "integrity": "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@huggingface/tokenizers": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", - "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", - "license": "Apache-2.0" - }, - "node_modules/@huggingface/transformers": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", - "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", - "license": "Apache-2.0", - "dependencies": { - "@huggingface/jinja": "^0.5.6", - "@huggingface/tokenizers": "^0.1.3", - "onnxruntime-node": "1.24.3", - "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", - "sharp": "^0.34.5" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", - "cpu": [ - "arm" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", - "cpu": [ - "riscv64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", - "cpu": [ - "arm" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", - "cpu": [ - "riscv64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.7.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, "node_modules/@ioredis/commands": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", @@ -805,63 +251,6 @@ "sparse-bitfield": "^3.0.3" } }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", - "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", - "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", - "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", - "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", - "license": "BSD-3-Clause" - }, "node_modules/@resvg/resvg-js": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz", @@ -1163,15 +552,6 @@ "node": ">= 0.6" } }, - "node_modules/adm-zip": { - "version": "0.5.18", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", - "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", - "license": "MIT", - "engines": { - "node": ">=12.0" - } - }, "node_modules/archiver": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", @@ -1349,13 +729,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/boolean": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", - "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", - "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", - "license": "MIT" - }, "node_modules/brace-expansion": { "version": "1.1.18", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", @@ -1702,40 +1075,6 @@ } } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/denque": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", @@ -1754,21 +1093,6 @@ "node": ">= 0.8" } }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/detect-node": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", - "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "license": "MIT" - }, "node_modules/dfa": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/dfa/-/dfa-1.2.0.tgz", @@ -1926,30 +1250,12 @@ "node": ">= 0.4" } }, - "node_modules/es6-error": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", - "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "license": "MIT" - }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "license": "MIT" }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -2103,12 +1409,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/flatbuffers": { - "version": "25.9.23", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", - "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", - "license": "Apache-2.0" - }, "node_modules/fn.name": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", @@ -2245,39 +1545,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/global-agent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", - "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", - "license": "BSD-3-Clause", - "dependencies": { - "boolean": "^3.0.1", - "es6-error": "^4.1.1", - "matcher": "^3.0.0", - "roarr": "^2.15.3", - "semver": "^7.3.2", - "serialize-error": "^7.0.1" - }, - "engines": { - "node": ">=10.0" - } - }, - "node_modules/globalthis": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", - "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "license": "MIT", - "dependencies": { - "define-properties": "^1.2.1", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -2296,24 +1563,6 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "license": "ISC" }, - "node_modules/guid-typescript": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", - "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", - "license": "ISC" - }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -2552,12 +1801,6 @@ "base64-js": "^1.5.1" } }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "license": "ISC" - }, "node_modules/jsonwebtoken": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", @@ -2893,24 +2136,6 @@ "node": ">= 12.0.0" } }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/matcher": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", - "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3206,15 +2431,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -3245,49 +2461,6 @@ "fn.name": "1.x.x" } }, - "node_modules/onnxruntime-common": { - "version": "1.24.3", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", - "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", - "license": "MIT" - }, - "node_modules/onnxruntime-node": { - "version": "1.24.3", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", - "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", - "hasInstallScript": true, - "license": "MIT", - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "adm-zip": "^0.5.16", - "global-agent": "^3.0.0", - "onnxruntime-common": "1.24.3" - } - }, - "node_modules/onnxruntime-web": { - "version": "1.26.0-dev.20260416-b7804b056c", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", - "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", - "license": "MIT", - "dependencies": { - "flatbuffers": "^25.1.24", - "guid-typescript": "^1.0.9", - "long": "^5.2.3", - "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", - "platform": "^1.3.6", - "protobufjs": "^7.2.4" - } - }, - "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.24.0-dev.20251116-b39e144322", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", - "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", - "license": "MIT" - }, "node_modules/openai": { "version": "7.5.0", "resolved": "https://registry.npmjs.org/openai/-/openai-7.5.0.tgz", @@ -3420,12 +2593,6 @@ "png-js": "^1.0.0" } }, - "node_modules/platform": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", - "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", - "license": "MIT" - }, "node_modules/png-js": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/png-js/-/png-js-1.1.0.tgz", @@ -3467,29 +2634,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, - "node_modules/protobufjs": { - "version": "7.6.6", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", - "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.5", - "@protobufjs/eventemitter": "^1.1.1", - "@protobufjs/fetch": "^1.1.1", - "@protobufjs/float": "^1.0.2", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.1", - "@types/node": ">=13.7.0", - "long": "^5.3.2" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3649,23 +2793,6 @@ "rimraf": "bin.js" } }, - "node_modules/roarr": { - "version": "2.15.4", - "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", - "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", - "license": "BSD-3-Clause", - "dependencies": { - "boolean": "^3.0.1", - "detect-node": "^2.0.4", - "globalthis": "^1.0.1", - "json-stringify-safe": "^5.0.1", - "semver-compare": "^1.0.0", - "sprintf-js": "^1.1.2" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/router": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", @@ -3750,12 +2877,6 @@ "node": ">=10" } }, - "node_modules/semver-compare": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", - "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "license": "MIT" - }, "node_modules/send": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", @@ -3782,21 +2903,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/serialize-error": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", - "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", - "license": "MIT", - "dependencies": { - "type-fest": "^0.13.1" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/serve-static": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", @@ -3828,50 +2934,6 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, - "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.0.0", - "detect-libc": "^2.1.2", - "semver": "^7.7.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" - } - }, "node_modules/side-channel": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", @@ -3959,12 +3021,6 @@ "memory-pager": "^1.0.2" } }, - "node_modules/sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "license": "BSD-3-Clause" - }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -4080,18 +3136,6 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, - "node_modules/type-fest": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", - "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/type-is": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", @@ -4304,27 +3348,6 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, - "node_modules/ws": { - "version": "8.21.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", - "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/xml": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", diff --git a/package.json b/package.json index 860652d..6a2c838 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "author": "WeLe EdTech", "license": "ISC", "dependencies": { - "@huggingface/transformers": "^4.2.0", "@langchain/core": "^1.1.18", "@langchain/langgraph": "^1.1.0", "@langchain/openai": "^1.5.10", @@ -27,12 +26,10 @@ "ioredis": "^5.10.1", "jsonwebtoken": "^9.0.2", "mongoose": "^9.2.3", - "onnxruntime-node": "^1.24.3", "pdfkit": "^0.17.2", "pptxgenjs": "^4.0.1", "uuid": "^13.0.0", "winston": "^3.19.0", - "ws": "^8.21.3", "zod": "^3.25.76" } } diff --git a/scripts/build-tamil-tokenizer.mjs b/scripts/build-tamil-tokenizer.mjs deleted file mode 100644 index f183970..0000000 --- a/scripts/build-tamil-tokenizer.mjs +++ /dev/null @@ -1,66 +0,0 @@ -/* Generate the tokenizer.json that Transformers.js needs for the exported - Tamil VITS model. - - `save_pretrained` does not emit one: VitsTokenizer is a "slow" tokenizer with - no fast counterpart, so Python writes vocab.json + tokenizer_config.json and - nothing else. Transformers.js only reads tokenizer.json, so we synthesise it - from the exported vocab, mirroring the structure of the working English - model (Xenova/mms-tts-eng) exactly. - - The four normalizer steps, in order: - 1. Lowercase — no-op for Tamil, matters for embedded Latin/digits - 2. Replace — drop every character outside the vocab - 3. Strip — trim surrounding whitespace - 4. Replace — insert the blank token between every character, - which is what `add_blank: true` means for VITS. - Omit this and the audio comes out garbled. -*/ -import fs from 'node:fs'; -import path from 'node:path'; - -const DIR = 'assets/tts/mms-tts-tam'; -const vocab = JSON.parse(fs.readFileSync(path.join(DIR, 'vocab.json'), 'utf8')); -const cfg = JSON.parse(fs.readFileSync(path.join(DIR, 'tokenizer_config.json'), 'utf8')); - -// The blank/pad token is whichever character maps to id 0. -const blank = Object.keys(vocab).find((k) => vocab[k] === 0); -const unk = cfg.unk_token ?? ''; -const unkId = vocab[unk] ?? Object.keys(vocab).length; - -// Character class of everything we keep. Escape the regex metacharacters that -// are still special inside a negated class. -const escaped = Object.keys(vocab) - .filter((c) => c !== unk) - .map((c) => (']\\^-'.includes(c) ? '\\' + c : c)) - .join(''); - -const tokenizer = { - version: '1.0', - truncation: null, - padding: null, - added_tokens: [{ - id: unkId, content: unk, - single_word: false, lstrip: false, rstrip: false, normalized: false, special: true, - }], - normalizer: { - type: 'Sequence', - normalizers: [ - { type: 'Lowercase' }, - { type: 'Replace', pattern: { Regex: `[^${escaped}]` }, content: '' }, - { type: 'Strip', strip_left: true, strip_right: true }, - ...(cfg.add_blank ? [{ type: 'Replace', pattern: { Regex: '(?=.)|(? None: - super().__init__() - self.m = m - - def forward(self, input_ids: torch.Tensor, attention_mask: torch.Tensor): - out = self.m(input_ids=input_ids, attention_mask=attention_mask) - return out.waveform, out.spectrogram - - -sample = tok("வணக்கம், இது ஒரு சோதனை.", return_tensors="pt") -fp32 = ONNX_DIR / "model.onnx" - -print("exporting to ONNX …") -torch.onnx.export( - Exportable(model), - (sample["input_ids"], sample["attention_mask"]), - str(fp32), - input_names=["input_ids", "attention_mask"], - output_names=["waveform", "spectrogram"], - dynamic_axes={ - "input_ids": {0: "batch", 1: "sequence"}, - "attention_mask": {0: "batch", 1: "sequence"}, - "waveform": {0: "batch", 1: "samples"}, - "spectrogram": {0: "batch", 2: "frames"}, - }, - opset_version=17, - do_constant_folding=True, -) -print(f" fp32: {fp32.stat().st_size / 1e6:.1f} MB") - -# ── int8 ──────────────────────────────────────────────────────────────────── -try: - from onnxruntime.quantization import QuantType, quantize_dynamic - - q = ONNX_DIR / "model_quantized.onnx" - quantize_dynamic(str(fp32), str(q), weight_type=QuantType.QUInt8) - print(f" int8: {q.stat().st_size / 1e6:.1f} MB") -except Exception as e: # noqa: BLE001 - print(f" quantisation skipped: {e}") - -# ── tokenizer + config, so the folder loads standalone ────────────────────── -tok.save_pretrained(OUT) -model.config.to_json_file(OUT / "config.json") - -# Transformers.js reads this to pick a default dtype. -(OUT / "quantize_config.json").write_text(json.dumps({"per_channel": False, "reduce_range": False}, indent=2)) - -print("\nwrote:") -for p in sorted(OUT.rglob("*")): - if p.is_file(): - print(f" {p.relative_to(OUT)} ({p.stat().st_size / 1e6:.2f} MB)") diff --git a/scripts/t-auto.mjs b/scripts/t-auto.mjs deleted file mode 100644 index 0e633d4..0000000 --- a/scripts/t-auto.mjs +++ /dev/null @@ -1,21 +0,0 @@ -/* Does auto mode now route Tamil to Tamil instead of silently using English? */ -import { synthesize, transcribe } from '../src/speech/index.js'; - -const resample = (a, from, to) => { - const r = from / to, out = new Float32Array(Math.floor(a.length / r)); - for (let i = 0; i < out.length; i++) { const p = i * r, k = Math.floor(p); out[i] = a[k] + (a[Math.min(k + 1, a.length - 1)] - a[k]) * (p - k); } - return out; -}; - -for (const [lang, text] of [ - ['ta', 'மூவாயிரம் நானூற்று இருபத்தேழு புதிய லீட்கள் உள்ளன.'], - ['en', 'How many new leads did we receive today?'], -]) { - const spoken = await synthesize(text, lang); - const audio = resample(spoken.audio, spoken.sampling_rate, 16000); - const r = await transcribe(audio, 'auto', 'ta'); - const ok = r.lang === lang ? 'PASS' : 'FAIL'; - console.log(`${ok} spoke ${lang} → routed ${r.lang} (detected ${r.detected} @ ${r.confidence}) ${r.ms}ms`); - console.log(` ${JSON.stringify(r.text.slice(0, 80))}`); -} -process.exit(0); diff --git a/scripts/t-detect.mjs b/scripts/t-detect.mjs deleted file mode 100644 index 2c0d9c4..0000000 --- a/scripts/t-detect.mjs +++ /dev/null @@ -1,62 +0,0 @@ -/* Can we get real language detection out of Whisper in Transformers.js? */ -import { AutoProcessor, WhisperForConditionalGeneration, Tensor, env } from '@huggingface/transformers'; -import { synthesize } from '../src/speech/index.js'; - -env.cacheDir = './.transformers-cache'; -const MODEL = 'onnx-community/whisper-base'; - -const processor = await AutoProcessor.from_pretrained(MODEL); -const model = await WhisperForConditionalGeneration.from_pretrained(MODEL, { dtype: 'q8' }); -const tok = processor.tokenizer; - -// Whisper emits one language token right after <|startoftranscript|>. Reading -// that distribution is a single decoder step — far cheaper than transcribing -// twice to see which language "looks better". -const id = (t) => tok.encode(t, { add_special_tokens: false })[0]; -const sot = id('<|startoftranscript|>'); -const CANDIDATES = ['en', 'ta']; -const langIds = CANDIDATES.map((c) => id(`<|${c}|>`)); -console.log('sot:', sot, '| language token ids:', JSON.stringify(Object.fromEntries(CANDIDATES.map((c, i) => [c, langIds[i]])))); - -function resample(a, from, to) { - const r = from / to; - const out = new Float32Array(Math.floor(a.length / r)); - for (let i = 0; i < out.length; i++) { - const p = i * r, k = Math.floor(p); - out[i] = a[k] + (a[Math.min(k + 1, a.length - 1)] - a[k]) * (p - k); - } - return out; -} - -for (const [lang, text] of [ - ['ta', 'மூவாயிரம் நானூற்று இருபத்தேழு புதிய லீட்கள் உள்ளன.'], - ['en', 'There are three thousand four hundred and twenty seven new leads.'], -]) { - const spoken = await synthesize(text, lang); - const audio = resample(spoken.audio, spoken.sampling_rate, 16000); - - const inputs = await processor(audio); - const t0 = Date.now(); - const out = await model({ - ...inputs, - decoder_input_ids: new Tensor('int64', BigInt64Array.from([BigInt(sot)]), [1, 1]), - }); - const ms = Date.now() - t0; - - const logits = out.logits; - const last = logits.dims[1] - 1; - const vocab = logits.dims[2]; - const row = logits.data.slice(last * vocab, (last + 1) * vocab); - - const scores = langIds.map((id) => Number(row[id])); - const max = Math.max(...scores); - const exp = scores.map((s) => Math.exp(s - max)); - const sum = exp.reduce((a, b) => a + b, 0); - const probs = exp.map((e) => e / sum); - const best = probs.indexOf(Math.max(...probs)); - - console.log(`spoken ${lang} → detected ${CANDIDATES[best]} ` - + `(${CANDIDATES.map((c, i) => `${c} ${probs[i].toFixed(3)}`).join(', ')}) in ${ms}ms ` - + `${CANDIDATES[best] === lang ? '✅' : '❌'}`); -} -process.exit(0); diff --git a/scripts/t-footprint.mjs b/scripts/t-footprint.mjs deleted file mode 100644 index 61dc6b2..0000000 --- a/scripts/t-footprint.mjs +++ /dev/null @@ -1,46 +0,0 @@ -/* English-only footprint: does it fit the 768 MB container cap on AWS? - - Loads exactly what an English-only deployment needs and reports RSS after - each stage, so the answer is measured rather than estimated. -*/ -const mb = () => Math.round(process.memoryUsage().rss / 1048576); -const step = (label) => console.log(` ${label.padEnd(34)} RSS ${String(mb()).padStart(4)} MB`); - -step('baseline (node + agent code)'); - -const { synthesize, transcribe, Endpointer } = await import('../src/speech/index.js'); -step('after importing speech module'); - -// VAD -const ep = new Endpointer(); -await ep.push(new Float32Array(16000)); -step('+ Silero VAD'); - -// TTS English -const spoken = await synthesize('There are three thousand four hundred and twenty seven new leads.', 'en'); -step('+ MMS-TTS English'); - -// STT -const resample = (a, from, to) => { - const r = from / to, out = new Float32Array(Math.floor(a.length / r)); - for (let i = 0; i < out.length; i++) { const p = i * r, k = Math.floor(p); out[i] = a[k] + (a[Math.min(k + 1, a.length - 1)] - a[k]) * (p - k); } - return out; -}; -const audio = resample(spoken.audio, spoken.sampling_rate, 16000); -const heard = await transcribe(audio, 'en', 'en'); -step('+ Whisper base (STT)'); - -// Steady state: a few turns, to see whether it keeps growing. -for (let i = 0; i < 3; i++) { - await synthesize('Checking the leads now.', 'en'); - await transcribe(audio, 'en', 'en'); -} -step('after 3 more turns'); - -console.log(`\n transcript: ${JSON.stringify(heard.text)}`); -console.log(` TTS rate : ${spoken.sampling_rate} Hz`); - -const peak = mb(); -const CAP = 768; -console.log(`\n peak ${peak} MB vs ${CAP} MB container cap → ${peak < CAP * 0.8 ? 'FITS ✅' : peak < CAP ? 'TIGHT ⚠️' : 'EXCEEDS ❌'}`); -process.exit(0); diff --git a/scripts/t-multiutterance.mjs b/scripts/t-multiutterance.mjs deleted file mode 100644 index d86a112..0000000 --- a/scripts/t-multiutterance.mjs +++ /dev/null @@ -1,85 +0,0 @@ -/* Reproduces the reported bug: first question works, later ones hang on - "Listening". - - Streams THREE spoken utterances over the real voice socket, at the same - 40 ms cadence the browser uses, with silence between them. Before the fix - the endpointer's state was corrupted by concurrent frame processing during - the first answer, so utterances 2 and 3 were never detected. -*/ -import WebSocket from 'ws'; -import jwt from 'jsonwebtoken'; -import dotenv from 'dotenv'; -import { synthesize } from '../src/speech/index.js'; - -dotenv.config(); -const URL_BASE = process.env.TEST_VOICE_URL || 'ws://localhost:4000'; -const token = jwt.sign({ id: '6a00cefe524bebd27037a968' }, process.env.CRM_JWT_SECRET, { expiresIn: '30m' }); - -const QUESTIONS = [ - 'How many leads are in the new lead stage?', - 'How many leads did we get today?', - 'Who has overdue follow ups?', -]; - -const resample = (a, from, to) => { - const r = from / to, o = new Float32Array(Math.floor(a.length / r)); - for (let i = 0; i < o.length; i++) { const p = i * r, k = Math.floor(p); o[i] = a[k] + (a[Math.min(k + 1, a.length - 1)] - a[k]) * (p - k); } - return o; -}; - -console.log('synthesising the three questions as speech…'); -const clips = []; -for (const q of QUESTIONS) { - const s = await synthesize(q, 'en'); - clips.push(resample(s.audio, s.sampling_rate, 16000)); -} - -const ws = new WebSocket(`${URL_BASE}/api/agent/voice?token=${encodeURIComponent(token)}`); -const transcripts = []; -let idleCount = 0; - -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); - -/** Send one clip at the browser's real cadence, then a second of silence. */ -async function speak(clip) { - const FRAME = 640; // 40 ms @16k - for (let i = 0; i < clip.length; i += FRAME) { - const slice = clip.subarray(i, Math.min(i + FRAME, clip.length)); - const pcm = Buffer.alloc(slice.length * 2); - for (let j = 0; j < slice.length; j++) { - const v = Math.max(-1, Math.min(1, slice[j])); - pcm.writeInt16LE(v < 0 ? v * 0x8000 : v * 0x7fff, j * 2); - } - ws.send(pcm); - await sleep(40); - } - const silence = Buffer.alloc(FRAME * 2); - for (let i = 0; i < 30; i++) { ws.send(silence); await sleep(40); } // 1.2 s -} - -ws.on('message', (d, bin) => { - if (bin) return; - const m = JSON.parse(d); - if (m.type === 'transcript') { transcripts.push(m.text); console.log(` 📝 ${transcripts.length}: ${JSON.stringify(m.text)}`); } - else if (m.type === 'idle') { idleCount++; console.log(` ✔ turn ${idleCount} complete`); } - else if (m.type === 'heard_nothing') console.log(' ⚠️ heard nothing'); - else if (m.type === 'error') console.log(' ✗ error:', m.message); -}); - -ws.on('open', async () => { - console.log('connected — streaming 3 utterances at browser cadence\n'); - for (let i = 0; i < clips.length; i++) { - console.log(`speaking #${i + 1}: ${JSON.stringify(QUESTIONS[i])}`); - await speak(clips[i]); - // Wait for this turn to finish before the next, as a person would. - const target = i + 1; - for (let w = 0; w < 120 && idleCount < target; w++) await sleep(1000); - } - - console.log(`\nRESULT: ${transcripts.length}/3 utterances detected, ${idleCount}/3 turns completed`); - console.log(transcripts.length === 3 ? '✅ PASS — later questions are heard' : '❌ FAIL — stuck after the first'); - ws.close(); - process.exit(transcripts.length === 3 ? 0 : 1); -}); - -ws.on('error', (e) => { console.log('socket error:', e.message); process.exit(1); }); diff --git a/scripts/t-speech.mjs b/scripts/t-speech.mjs deleted file mode 100644 index a66fd98..0000000 --- a/scripts/t-speech.mjs +++ /dev/null @@ -1,44 +0,0 @@ -/* Do Whisper (STT) and MMS-TTS (TTS) actually run in Node on CPU? */ -import { pipeline, env } from '@huggingface/transformers'; - -env.cacheDir = './.transformers-cache'; - -const t = (t0) => `${((performance.now() - t0) / 1000).toFixed(1)}s`; - -// ── TTS: MMS-TTS Tamil (VITS, 36M, feed-forward) ──────────────────────────── -console.log('[1/2] loading MMS-TTS Tamil…'); -let t0 = performance.now(); -const tts = await pipeline('text-to-speech', 'Xenova/mms-tts-eng', { dtype: 'fp32' }); -console.log(` loaded in ${t(t0)}`); - -const TA = 'There are three thousand four hundred leads in the new lead stage.'; -await tts(TA); // warm -for (const [label, text] of [['short', TA], ['long', TA + ' ' + TA + ' ' + TA]]) { - t0 = performance.now(); - const out = await tts(text); - const ms = performance.now() - t0; - const audioMs = (out.audio.length / out.sampling_rate) * 1000; - console.log( - ` ${label.padEnd(5)} ${String(text.length).padStart(3)} chars → ${ms.toFixed(0)}ms ` - + `for ${audioMs.toFixed(0)}ms audio @ ${out.sampling_rate}Hz → RTF ${(ms / audioMs).toFixed(2)}x`, - ); -} - -// ── STT: Whisper (multilingual — Tamil, Hindi, English + detection) ───────── -console.log('\n[2/2] loading Whisper base…'); -t0 = performance.now(); -const stt = await pipeline('automatic-speech-recognition', 'onnx-community/whisper-base', { dtype: 'q8' }); -console.log(` loaded in ${t(t0)}`); - -// 4 s of quiet noise — proves the graph runs and times it. -const audio = Float32Array.from({ length: 16000 * 4 }, () => (Math.random() - 0.5) * 0.02); -t0 = performance.now(); -const r = await stt(audio, { language: 'ta', task: 'transcribe' }); -console.log(` 4000ms audio → ${(performance.now() - t0).toFixed(0)}ms → ${JSON.stringify(r.text).slice(0, 60)}`); - -t0 = performance.now(); -const r2 = await stt(audio, { language: 'en', task: 'transcribe' }); -console.log(` english pass → ${(performance.now() - t0).toFixed(0)}ms → ${JSON.stringify(r2.text).slice(0, 60)}`); - -console.log(`\nRSS ${(process.memoryUsage().rss / 1e9).toFixed(2)} GB`); -process.exit(0); diff --git a/scripts/t-tamil-tts.mjs b/scripts/t-tamil-tts.mjs deleted file mode 100644 index 9d04e45..0000000 --- a/scripts/t-tamil-tts.mjs +++ /dev/null @@ -1,55 +0,0 @@ -/* Does the locally-exported Tamil ONNX load and speak through Transformers.js? */ -import { pipeline, env } from '@huggingface/transformers'; -import fs from 'node:fs'; - -// Load from the local folder, not the Hub. -env.allowRemoteModels = false; -env.localModelPath = './assets/tts'; - -for (const dtype of ['q8', 'fp32']) { - try { - const t0 = performance.now(); - const tts = await pipeline('text-to-speech', 'mms-tts-tam', { dtype }); - const load = performance.now() - t0; - - const TEXT = 'மூவாயிரம் நானூற்று இருபத்தேழு புதிய லீட்கள் உள்ளன.'; - await tts(TEXT); // warm - - const t1 = performance.now(); - const out = await tts(TEXT); - const ms = performance.now() - t1; - const audioMs = (out.audio.length / out.sampling_rate) * 1000; - - console.log( - `${dtype.padEnd(5)} load ${(load / 1000).toFixed(1)}s | ` - + `${ms.toFixed(0)}ms for ${audioMs.toFixed(0)}ms audio @ ${out.sampling_rate}Hz | ` - + `RTF ${(ms / audioMs).toFixed(2)}x`, - ); - - // Non-silent output is the real proof the graph is wired correctly. - const peak = out.audio.reduce((m, v) => Math.max(m, Math.abs(v)), 0); - console.log(` samples ${out.audio.length}, peak amplitude ${peak.toFixed(3)} ${peak > 0.01 ? '✅ audible' : '⚠️ SILENT'}`); - - if (dtype === 'q8') { - const wav = toWav(out.audio, out.sampling_rate); - fs.writeFileSync('scripts/tamil-sample.wav', wav); - console.log(' wrote scripts/tamil-sample.wav — play it to judge quality'); - } - } catch (e) { - console.log(`${dtype.padEnd(5)} FAILED: ${e.message.slice(0, 160)}`); - } -} - -function toWav(samples, rate) { - const buf = Buffer.alloc(44 + samples.length * 2); - buf.write('RIFF', 0); buf.writeUInt32LE(36 + samples.length * 2, 4); buf.write('WAVE', 8); - buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22); - buf.writeUInt32LE(rate, 24); buf.writeUInt32LE(rate * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34); - buf.write('data', 36); buf.writeUInt32LE(samples.length * 2, 40); - for (let i = 0; i < samples.length; i++) { - const s = Math.max(-1, Math.min(1, samples[i])); - buf.writeInt16LE(s < 0 ? s * 0x8000 : s * 0x7fff, 44 + i * 2); - } - return buf; -} -process.exit(0); diff --git a/scripts/t-voice.mjs b/scripts/t-voice.mjs deleted file mode 100644 index 5e5e223..0000000 --- a/scripts/t-voice.mjs +++ /dev/null @@ -1,72 +0,0 @@ -/* End-to-end check of the in-process speech pipeline: TTS → VAD → STT. - - Synthesising a sentence and feeding that audio back through the endpointer - and recogniser exercises every stage with real speech, which a noise buffer - cannot do — silence never opens a VAD turn. -*/ -import { synthesize, transcribe, sentences, speakable, Endpointer } from '../src/speech/index.js'; - -const say = (m) => console.log(m); - -// ── 1. TTS both languages ─────────────────────────────────────────────────── -const CASES = [ - ['ta', 'மூவாயிரம் நானூற்று இருபத்தேழு புதிய லீட்கள் உள்ளன.'], - ['en', 'There are three thousand four hundred and twenty seven new leads.'], -]; - -const rendered = {}; -for (const [lang, text] of CASES) { - const t0 = Date.now(); - await synthesize(text, lang); // warm - const t1 = Date.now(); - const out = await synthesize(text, lang); - const ms = Date.now() - t1; - const audioMs = (out.audio.length / out.sampling_rate) * 1000; - rendered[lang] = out; - say(`TTS ${lang} warm ${((t1 - t0) / 1000).toFixed(1)}s | ${ms}ms for ${audioMs.toFixed(0)}ms ` - + `@${out.sampling_rate}Hz | RTF ${(ms / audioMs).toFixed(2)}x`); -} - -// ── 2. VAD: does synthesised speech open and close a turn? ────────────────── -function resample(audio, from, to) { - if (from === to) return audio; - const ratio = from / to; - const out = new Float32Array(Math.floor(audio.length / ratio)); - for (let i = 0; i < out.length; i++) { - const p = i * ratio; - const a = Math.floor(p); - out[i] = audio[a] + (audio[Math.min(a + 1, audio.length - 1)] - audio[a]) * (p - a); - } - return out; -} - -const ep = new Endpointer(); -const speech = resample(rendered.en.audio, rendered.en.sampling_rate, 16000); -// Speech, then a second of silence so the endpointer closes the turn. -const withTail = new Float32Array(speech.length + 16000); -withTail.set(speech); - -let started = false; -let captured = null; -for (let i = 0; i < withTail.length; i += 640) { // 40 ms chunks, as the browser sends - const { utterances, started: s } = await ep.push(withTail.subarray(i, Math.min(i + 640, withTail.length))); - if (s) started = true; - if (utterances.length) { captured = utterances[0]; break; } -} -say(`VAD speech detected: ${started ? 'yes' : 'NO'} | turn closed: ${captured ? 'yes' : 'NO'}` - + (captured ? ` | captured ${(captured.length / 16000).toFixed(2)}s` : '')); - -// ── 3. STT on that captured audio ─────────────────────────────────────────── -if (captured) { - for (const lang of ['en', 'auto']) { - const r = await transcribe(captured, lang, 'en'); - say(`STT ${lang.padEnd(4)} ${r.ms}ms → lang=${r.lang}${r.detected ? ` (heard ${r.detected})` : ''} → ${JSON.stringify(r.text.slice(0, 70))}`); - } -} - -// ── 4. Text shaping ───────────────────────────────────────────────────────── -const md = '## Leads\n\n**3,427** in `new_lead`.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n- Only 12% contacted.\nNext step is triage.'; -say(`\nspeakable: ${JSON.stringify(speakable(md))}`); -say(`sentences: ${JSON.stringify(sentences(speakable(md)))}`); -say(`\nRSS ${(process.memoryUsage().rss / 1e9).toFixed(2)} GB`); -process.exit(0); diff --git a/src/gateway/voice.js b/src/gateway/voice.js deleted file mode 100644 index 32c75ef..0000000 --- a/src/gateway/voice.js +++ /dev/null @@ -1,287 +0,0 @@ -// ============================================ -// Voice channel — speech in, speech out, in this same Node process. -// -// Voice is a *channel*, not a parallel product: a spoken question runs through -// the same graph, guardrails and agents as a typed one. Only the transport and -// the presentation differ, which is why this file contains no CRM logic. -// -// browser ──PCM16──► Endpointer ──► transcribe() ──► runTurn(graph) -// │ │ -// browser ◄──float32──── synthesize() ◄── sentences ◄─────┘ -// -// The hard problem is not audio, it is that a turn takes 17–46 s and that is -// dead silence in voice. So this speaks an acknowledgement within ~1 s, -// narrates each agent delegation aloud, then reads the answer sentence by -// sentence as it is composed. -// ============================================ -import { WebSocketServer, WebSocket } from 'ws'; -import { principalFromToken } from './auth.js'; -import { runTurn } from '../orchestration/runner.js'; -import { - Endpointer, transcribe, synthesize, sentences, speakable, LANGUAGES, -} from '../speech/index.js'; -import config from '../config/index.js'; -import logger from '../utils/logger.js'; - -/** Spoken filler, said the instant a question lands. */ -const ACK = { - ta: ['பார்க்கிறேன்.', 'ஒரு நிமிடம், பார்க்கிறேன்.'], - en: ['Let me check.', 'One moment, checking now.'], -}; - -/** Progress narration — spoken over the user's waiting time, so keep it short. */ -const NARRATE = { - ta: { lead: 'லீட் விவரங்களைப் பார்க்கிறேன்.', analytics: 'புள்ளிவிவரங்களைச் சரிபார்க்கிறேன்.', conversation: 'உரையாடல்களைப் பார்க்கிறேன்.', default: 'தரவைச் சரிபார்க்கிறேன்.' }, - en: { lead: 'Checking the leads.', analytics: 'Pulling the numbers.', conversation: 'Looking at the conversations.', default: 'Checking the data.' }, -}; - -const NOTHING = { ta: 'பதில் கிடைக்கவில்லை.', en: 'I could not find an answer for that.' }; -const OOPS = { ta: 'மன்னிக்கவும், ஒரு பிழை ஏற்பட்டது.', en: 'Sorry, something went wrong.' }; - -const pick = (a) => a[Math.floor(Math.random() * a.length)]; -const ackFor = (l) => pick(ACK[l] || ACK.en); -const narrateFor = (l, agent) => (NARRATE[l] || NARRATE.en)[agent] || (NARRATE[l] || NARRATE.en).default; - -class VoiceSession { - constructor(client, user) { - this.client = client; - this.user = user; - this.lang = 'auto'; // what the user selected - this.replyLang = 'ta'; // what the last utterance actually was - this.prefer = 'ta'; // tiebreak when detection is unusable - this.sessionId = `voice:${Date.now().toString(36)}:${Math.random().toString(36).slice(2, 8)}`; - this.endpointer = new Endpointer(); - this.busy = false; - this.abort = null; - this.speakSeq = 0; // rising token; stale synthesis is discarded - this.narrated = new Set(); - // Audio frames are processed strictly one at a time. The endpointer holds - // recurrent VAD state plus a partial-frame buffer, and neither survives - // concurrent access — see onAudio(). - this.audioChain = Promise.resolve(); - // Turns are serialized too. handleUtterance() is fire-and-forget so the - // audio queue keeps flowing, which means two utterances can overlap; this - // chain guarantees one answer at a time without dropping the second. - this.turnChain = Promise.resolve(); - } - - send(obj) { - if (this.client.readyState === WebSocket.OPEN) this.client.send(JSON.stringify(obj)); - } - - sendAudio(buf) { - if (this.client.readyState === WebSocket.OPEN) this.client.send(buf, { binary: true }); - } - - // ── microphone ─────────────────────────────────────────────────────────── - /** - * The browser streams a frame every 40 ms and the socket's 'message' handler - * does not await us, so without a queue ~25 calls a second would run - * concurrently against one Endpointer — interleaving its `pending` buffer and - * Silero's recurrent state until it stopped detecting speech at all. That is - * exactly what made the FIRST question work and every one after it hang on - * "Listening": the state was corrupted while the first answer was running. - */ - onAudio(data) { - this.audioChain = this.audioChain - .then(() => this.processAudio(data)) - .catch((e) => logger.error(`audio frame failed: ${e.message}`)); - return this.audioChain; - } - - async processAudio(data) { - // Browser sends 16 kHz mono PCM16; the models want float32 in [-1, 1]. - const pcm16 = new Int16Array(data.buffer, data.byteOffset, Math.floor(data.byteLength / 2)); - const pcm = new Float32Array(pcm16.length); - for (let i = 0; i < pcm16.length; i++) pcm[i] = pcm16[i] / 32768; - - let result; - try { - result = await this.endpointer.push(pcm); - } catch (e) { - logger.error(`VAD failed: ${e.message}`); - this.send({ type: 'error', message: 'Voice input failed to initialise. Check the server logs.' }); - return; - } - - if (result.started) { - // Barge-in: the user talking wins immediately. Bumping the token drops - // any in-flight synthesis rather than letting it arrive late. - this.speakSeq++; - this.abort?.abort(); - this.send({ type: 'barge_in' }); - } - - // Deliberately NOT awaited: a turn takes 17-46 s, and awaiting it here - // would stall the audio queue for that whole time — no barge-in, and a - // backlog of frames to grind through afterwards. - for (const utterance of result.utterances) { - this.handleUtterance(utterance).catch((e) => logger.error(`turn failed: ${e.message}`)); - } - } - - async handleUtterance(audio) { - let heard; - try { - heard = await transcribe(audio, this.lang, this.prefer); - } catch (e) { - logger.error(`STT failed: ${e.message}`); - this.send({ type: 'error', message: 'Could not transcribe that. Try again.' }); - return; - } - - if (!heard.text) { - this.send({ type: 'heard_nothing' }); - return; - } - - this.replyLang = heard.lang; - this.send({ type: 'transcript', text: heard.text, lang: heard.lang, detected: heard.detected, ms: heard.ms }); - - // Cut the running turn short so the new question is answered promptly - // rather than queueing behind 40 s of superseded work. - if (this.busy) this.abort?.abort(); - - this.turnChain = this.turnChain - .then(() => this.answer(heard.text)) - .catch((e) => logger.error(`turn failed: ${e.message}`)); - await this.turnChain; - } - - // ── speaking ───────────────────────────────────────────────────────────── - /** Synthesise and stream one piece, unless a newer turn has superseded it. */ - async say(text, seq) { - const clean = speakable(text); - if (!clean || seq !== this.speakSeq) return; - try { - const out = await synthesize(clean, this.replyLang); - if (!out || seq !== this.speakSeq) return; // interrupted while generating - - this.send({ type: 'audio_start', sample_rate: out.sampling_rate }); - // Float32 straight down the socket — the playback worklet takes it as-is. - this.sendAudio(Buffer.from(out.audio.buffer, out.audio.byteOffset, out.audio.byteLength)); - this.send({ type: 'audio_end' }); - } catch (e) { - logger.error(`TTS failed: ${e.message}`); - } - } - - async answer(question) { - this.busy = true; - this.narrated.clear(); - this.abort = new AbortController(); - const seq = ++this.speakSeq; - - // Answer the silence immediately. The pipeline still takes 17–46 s, but - // the user hears a response in about a second. - this.say(ackFor(this.replyLang), seq); - this.send({ type: 'thinking' }); - - try { - const result = await runTurn({ - sessionId: this.sessionId, - message: question, - user: this.user, - channel: 'crm_chat', // voice users are staff - signal: this.abort.signal, - onEvent: (ev) => { - this.send(ev); - // Narrate delegations, once per agent, or it chatters. - if (ev.type === 'step' && ev.kind === 'delegate') { - const agent = String(ev.label || '').toLowerCase().split(' ')[0]; - if (!this.narrated.has(agent)) { - this.narrated.add(agent); - this.say(narrateFor(this.replyLang, agent), seq); - } - } - }, - }); - - this.send({ type: 'result', blocks: result.blocks, usage: result.usage }); - - const answer = (result.blocks || []) - .filter((b) => b.type === 'text').map((b) => b.markdown).join(' ') || result.answer || ''; - const parts = sentences(speakable(answer)); - - if (!parts.length) { - await this.say(NOTHING[this.replyLang] || NOTHING.en, seq); - } else { - // Sequential on purpose: parallel synthesis would race to the socket - // and play the answer out of order. - for (const part of parts) { - if (seq !== this.speakSeq || this.abort.signal.aborted) break; - await this.say(part, seq); - } - } - } catch (err) { - if (err?.name !== 'AbortError') { - logger.error(`voice turn failed: ${err.message}`); - await this.say(OOPS[this.replyLang] || OOPS.en, seq); - } - } finally { - this.busy = false; - this.send({ type: 'idle' }); - } - } - - // ── control ────────────────────────────────────────────────────────────── - onMessage(data, isBinary) { - if (isBinary) return this.onAudio(data); - - let msg; - try { msg = JSON.parse(data.toString()); } catch { return undefined; } - - if (msg.type === 'config' && msg.lang) { - this.lang = msg.lang; - if (msg.lang !== 'auto') this.replyLang = this.prefer = msg.lang; - else if (msg.prefer) this.prefer = msg.prefer; - this.endpointer.reset(); - this.send({ type: 'config_ok', lang: this.lang, prefer: this.prefer }); - } else if (msg.type === 'cancel') { - this.speakSeq++; - this.abort?.abort(); - this.send({ type: 'cancelled' }); - } else if (msg.type === 'text' && msg.text) { - this.send({ type: 'transcript', text: msg.text, lang: this.replyLang, typed: true }); - this.answer(msg.text); - } - return undefined; - } - - close() { - this.speakSeq++; - this.abort?.abort(); - } -} - -/** Attach the voice WebSocket to the HTTP server. */ -export function attachVoice(server) { - const wss = new WebSocketServer({ noServer: true }); - - server.on('upgrade', async (req, socket, head) => { - const url = new URL(req.url, `http://${req.headers.host}`); - if (url.pathname !== '/api/agent/voice') return; // leave other upgrades alone - - // Browsers cannot set headers on a WebSocket, so the CRM token arrives as - // a query parameter. Same token, same verification as every other route. - const user = await principalFromToken(url.searchParams.get('token')).catch(() => null); - if (!user) { - socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n'); - socket.destroy(); - return; - } - - wss.handleUpgrade(req, socket, head, (client) => { - const session = new VoiceSession(client, user); - logger.info(`🎙️ voice session ${session.sessionId} (${user.name})`); - session.send({ type: 'ready', session_id: session.sessionId, languages: LANGUAGES }); - - client.on('message', (d, bin) => session.onMessage(d, bin)); - client.on('close', () => session.close()); - client.on('error', () => session.close()); - }); - }); - - logger.info(` voice =ws://localhost:${config.port}/api/agent/voice (in-process, CPU)`); - return wss; -} diff --git a/src/server.js b/src/server.js index 3f9841b..9a613c0 100644 --- a/src/server.js +++ b/src/server.js @@ -16,8 +16,6 @@ import agentRoutes, { artifactRouter } from './gateway/routes.js'; import { ensureDir, sweep } from './output/artifactStore.js'; import crmApi from './tools/http/crmApi.js'; import { describeChains } from './orchestration/llm.js'; -import { attachVoice } from './gateway/voice.js'; -import { warmup as warmSpeech, speechStatus } from './speech/index.js'; const app = express(); @@ -41,7 +39,6 @@ app.get('/health', async (_req, res) => { redis: redisOk ? redisMode() : 'unavailable', crm_api: crm.reachable ? 'reachable' : `unreachable (${crm.error || crm.status})`, models: describeChains(), - speech: speechStatus(), uptime_s: Math.round(process.uptime()), }); }); @@ -81,15 +78,6 @@ async function start() { logger.info(` CRM API =${config.crmApi.base}`); }); - // Voice is a WebSocket upgrade on the same port, so the browser needs no - // second origin and the CRM token works unchanged. - attachVoice(server); - - // Speech models load lazily on the first voice turn (~10 s). Set - // SPEECH_WARMUP=true to pay that at boot instead — worth it in production, - // wasteful in development where most restarts never use voice. - if (process.env.SPEECH_WARMUP === 'true') warmSpeech(['ta', 'en']); - const shutdown = (sig) => { logger.info(`${sig} — shutting down`); server.close(() => process.exit(0)); diff --git a/src/speech/index.js b/src/speech/index.js deleted file mode 100644 index b95a4a0..0000000 --- a/src/speech/index.js +++ /dev/null @@ -1,181 +0,0 @@ -// ============================================ -// Speech pipeline — transcribe() and synthesize(). -// -// Whisper is multilingual and can identify the spoken language, so "auto" -// costs nothing extra: detection and transcription are the same forward pass. -// That matters for a WeLe agent who switches between Tamil and English inside -// one shift and should never have to touch a language menu. -// ============================================ -import { Tensor } from '@huggingface/transformers'; -import { getSTT, getTTS, supportsTTS, sttIsEnglishOnly, defaultLanguage } from './models.js'; -import logger from '../utils/logger.js'; - -export { LANGUAGES, warmup, speechStatus, defaultLanguage } from './models.js'; -export { Endpointer, warmupVad } from './vad.js'; - -const RATE = 16000; - -/** Languages we can both hear and speak. */ -const SPOKEN = new Set(['ta', 'en']); - -// Below this, trust the caller's preference over the detector. Short or noisy -// utterances — and code-mixed "Tanglish" especially — can land either side. -const DETECT_CONFIDENCE = Number(process.env.DETECT_CONFIDENCE ?? 0.6); - -let detectIds = null; - -/** - * Identify the spoken language in ONE decoder step. - * - * Passing no `language` to the pipeline does NOT auto-detect — Transformers.js - * logs "No language specified - defaulting to English" and transcribes Tamil - * as English, producing nonsense. Whisper does emit a language token right - * after <|startoftranscript|>, so we read that distribution directly. Measured - * ~700 ms, and 0.998 / 1.000 confidence on clean Tamil / English. - * - * Reuses the pipeline's own model and processor, so nothing loads twice. - */ -async function detectLanguage(audio) { - const stt = await getSTT(); - const tok = stt.tokenizer; - - if (!detectIds) { - const id = (t) => tok.encode(t, { add_special_tokens: false })[0]; - detectIds = { sot: id('<|startoftranscript|>'), langs: [...SPOKEN].map((c) => ({ code: c, id: id(`<|${c}|>`) })) }; - } - - const inputs = await stt.processor(audio); - const out = await stt.model({ - ...inputs, - decoder_input_ids: new Tensor('int64', BigInt64Array.from([BigInt(detectIds.sot)]), [1, 1]), - }); - - const { dims, data } = out.logits; - const row = data.slice((dims[1] - 1) * dims[2], dims[1] * dims[2]); - const scores = detectIds.langs.map((l) => Number(row[l.id])); - const max = Math.max(...scores); - const exp = scores.map((v) => Math.exp(v - max)); - const sum = exp.reduce((a, b) => a + b, 0); - const probs = exp.map((v) => v / sum); - const best = probs.indexOf(Math.max(...probs)); - - return { lang: detectIds.langs[best].code, confidence: probs[best] }; -} - -/** - * @param {Float32Array} audio mono @16 kHz in [-1, 1] - * @param {string} lang 'auto' | 'ta' | 'en' - * @param {string} prefer used when detection is unusable - */ -export async function transcribe(audio, lang = 'auto', prefer = defaultLanguage()) { - if (!audio || audio.length < RATE / 5) { // under 200 ms - return { text: '', lang: prefer, note: 'too short' }; - } - - const stt = await getSTT(); - const t0 = Date.now(); - - // Whisper must always be told a language — it never detects on its own here. - let used = lang; - let detected = null; - let confidence = null; - - // An English-only checkpoint has no language tokens to read, and passing - // `language` to it is rejected — so "auto" simply means English there. - if (lang === 'auto' && sttIsEnglishOnly()) { - used = 'en'; - } else if (lang === 'auto') { - try { - const d = await detectLanguage(audio); - detected = d.lang; - confidence = d.confidence; - used = d.confidence >= DETECT_CONFIDENCE ? d.lang : prefer; - if (used !== d.lang) { - logger.info(`language ID unsure (${d.lang} @ ${d.confidence.toFixed(2)}) — using preferred ${prefer}`); - } - } catch (e) { - logger.warn(`language ID failed (${e.message}) — using preferred ${prefer}`); - used = prefer; - } - } - - // An English-only checkpoint rejects BOTH `task` and `language` — it has no - // other mode to select. Multilingual builds require the language, since they - // silently default to English otherwise. - const opts = { return_timestamps: false }; - if (!sttIsEnglishOnly()) { - opts.task = 'transcribe'; - opts.language = used; - } - const result = await stt(audio, opts); - return finish(result, used, audio, t0, detected, confidence); -} - -function finish(result, used, audio, t0, detected, confidence) { - const text = (result?.text || '').trim(); - const ms = Date.now() - t0; - const audioMs = Math.round((audio.length / RATE) * 1000); - logger.info(`🎤 STT ${used}${detected && detected !== used ? ` (heard ${detected})` : ''}: ${audioMs}ms → ${ms}ms → ${JSON.stringify(text.slice(0, 70))}`); - return { - text, lang: used, detected: detected || null, - confidence: confidence == null ? null : Number(confidence.toFixed(3)), - ms, audio_ms: audioMs, - }; -} - -/** - * Synthesise one piece of text. - * @returns {Promise<{audio: Float32Array, sampling_rate: number}>} - */ -export async function synthesize(text, lang = defaultLanguage()) { - const clean = (text || '').trim(); - if (!clean) return null; - - const use = supportsTTS(lang) ? lang : defaultLanguage(); - const tts = await getTTS(use); - - const t0 = Date.now(); - const out = await tts(clean); - const ms = Date.now() - t0; - const audioMs = (out.audio.length / out.sampling_rate) * 1000; - logger.debug(`🔈 TTS ${use}: ${clean.length} chars → ${ms}ms for ${audioMs.toFixed(0)}ms (RTF ${(ms / audioMs).toFixed(2)}x)`); - - return { audio: out.audio, sampling_rate: out.sampling_rate }; -} - -/** - * Split into speakable pieces. Short prompts reach audio sooner, and a sentence - * boundary is a clean place to be interrupted. - */ -export function sentences(text, max = 200) { - const out = []; - for (const raw of String(text || '').split(/(?<=[.!?।])\s+/)) { - let s = raw.trim(); - if (!s) continue; - while (s.length > max) { - const cut = s.lastIndexOf(' ', max); - out.push(s.slice(0, cut > 0 ? cut : max).trim()); - s = s.slice(cut > 0 ? cut : max).trim(); - } - if (s) out.push(s); - } - return out; -} - -/** - * Strip block markdown before speaking — tables and code read terribly aloud, - * and the visual blocks are already on screen. - */ -export function speakable(markdown = '') { - return markdown - .replace(/```[\s\S]*?```/g, ' ') - .replace(/^\s*\|.*\|\s*$/gm, ' ') - .replace(/^\s*[-*]\s+/gm, '') - .replace(/^#{1,6}\s*/gm, '') - .replace(/\*\*([^*]+)\*\*/g, '$1') - .replace(/`([^`]+)`/g, '$1') - .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') - .replace(/₹\s?([\d,.]+)/g, 'rupees $1') - .replace(/\s{2,}/g, ' ') - .trim(); -} diff --git a/src/speech/models.js b/src/speech/models.js deleted file mode 100644 index 85d5d73..0000000 --- a/src/speech/models.js +++ /dev/null @@ -1,137 +0,0 @@ -// ============================================ -// Speech models — all ONNX, all CPU, all in this Node process. -// -// There is no GPU and no Python. That is the whole point: the AWS host has -// neither, and a second service was one more thing to deploy and keep alive. -// -// VAD Silero 2 MB endpointing -// STT Whisper base ~80 MB Tamil + English + language detection -// TTS MMS-TTS VITS ~114 MB per language, feed-forward -// -// Measured on an i7-10850H, CPU only: -// TTS RTF 0.28x (3.5x faster than realtime) -// STT ~1.2 s for 4 s of audio -// -// Two findings worth keeping: -// -// * VITS is feed-forward. The earlier Parler-TTS attempt was autoregressive -// and ran at RTF ~5x — i.e. 5x SLOWER than realtime — which is why voice was -// unusable even on a GPU. Architecture mattered far more than hardware here. -// -// * int8 is a trap for a model this small: dynamic quantisation made TTS 5.7x -// SLOWER than fp32 (RTF 1.67x vs 0.28x) because the quantise/dequantise -// overhead dominates. We ship fp32 deliberately. -// ============================================ -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { pipeline, env } from '@huggingface/transformers'; -import logger from '../utils/logger.js'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); - -// Hub downloads are cached here so a container restart does not re-fetch. -env.cacheDir = process.env.SPEECH_CACHE_DIR || path.join(ROOT, '.transformers-cache'); -// Tamil is loaded from a folder we exported ourselves — no public ONNX build -// of mms-tts-tam exists. See scripts/export-tamil-tts.py. -env.localModelPath = path.join(ROOT, 'assets/tts'); - -// whisper-tiny.en by default: measured, Whisper is the memory hog, not TTS. -// base cost 554 MB of an 879 MB total, which overran the 768 MB container cap. -// The .en build is half the size and, being English-only, cannot detect a -// language — which is fine when VOICE_LANGUAGES is just `en`. -const STT_MODEL = process.env.STT_MODEL || 'onnx-community/whisper-tiny.en'; - -/** True when the STT checkpoint is English-only and cannot identify languages. */ -export const sttIsEnglishOnly = () => /\.en$/.test(STT_MODEL); - -/** TTS voice per language. Tamil is exported locally; English is on the Hub. */ -const ALL_VOICES = { - en: { id: 'Xenova/mms-tts-eng', local: false, label: 'English', native: 'English' }, - ta: { id: 'mms-tts-tam', local: true, label: 'Tamil', native: 'தமிழ்' }, -}; - -// Each extra language is a further ~200 MB resident. Enable only what the -// deployment actually speaks — English alone on the current AWS box. -const ENABLED = (process.env.VOICE_LANGUAGES || 'en') - .split(',').map((s) => s.trim()).filter((c) => ALL_VOICES[c]); - -const VOICES = Object.fromEntries(ENABLED.map((c) => [c, ALL_VOICES[c]])); - -export const LANGUAGES = [ - // Auto-detect is only offered when there is a choice to make AND the STT - // model can actually detect — offering it otherwise is a lie. - ...(ENABLED.length > 1 && !sttIsEnglishOnly() - ? [{ code: 'auto', label: 'Auto-detect', native: 'Auto' }] : []), - ...ENABLED.map((c) => ({ code: c, label: ALL_VOICES[c].label, native: ALL_VOICES[c].native })), -]; - -export const defaultLanguage = () => (LANGUAGES[0]?.code || 'en'); - -const cache = new Map(); -let sttPromise = null; - -/** - * Models load on first use, not at boot. A CRM restart should not wait ~10 s - * for speech models that most sessions never touch. - */ -async function loadOnce(key, build) { - if (!cache.has(key)) { - const t0 = Date.now(); - cache.set(key, build().then((m) => { - logger.info(`🔊 loaded ${key} in ${((Date.now() - t0) / 1000).toFixed(1)}s`); - return m; - }).catch((e) => { - cache.delete(key); // let the next attempt retry - throw e; - })); - } - return cache.get(key); -} - -export async function getSTT() { - if (!sttPromise) { - sttPromise = loadOnce(STT_MODEL, () => - // q8 is the right call for Whisper — unlike VITS it is big enough that - // quantisation is a clear win. - pipeline('automatic-speech-recognition', STT_MODEL, { dtype: 'q8' }), - ).catch((e) => { sttPromise = null; throw e; }); - } - return sttPromise; -} - -export async function getTTS(lang) { - const voice = VOICES[lang] || VOICES[ENABLED[0]]; - const prev = env.allowRemoteModels; - try { - // Local folders must not be looked up on the Hub, and vice versa. - env.allowRemoteModels = !voice.local; - return await loadOnce(`tts:${voice.id}`, () => - pipeline('text-to-speech', voice.id, { dtype: 'fp32' }), - ); - } finally { - env.allowRemoteModels = prev; - } -} - -export const supportsTTS = (lang) => Boolean(VOICES[lang]); - -/** Warm the models the deployment actually expects to use. */ -export async function warmup(langs = ENABLED) { - try { - await getSTT(); - for (const l of langs) await getTTS(l); - logger.info('🔊 speech models warm'); - } catch (e) { - logger.warn(`speech warmup failed (will retry on first use): ${e.message}`); - } -} - -export function speechStatus() { - return { - stt_model: STT_MODEL, - english_only_stt: sttIsEnglishOnly(), - tts_voices: Object.fromEntries(Object.entries(VOICES).map(([k, v]) => [k, v.id])), - loaded: [...cache.keys()], - languages: LANGUAGES, - }; -} diff --git a/src/speech/vad.js b/src/speech/vad.js deleted file mode 100644 index 1f8077a..0000000 --- a/src/speech/vad.js +++ /dev/null @@ -1,152 +0,0 @@ -// ============================================ -// Endpointing — Silero VAD via onnxruntime-node. -// -// Deciding turn boundaries on the server rather than in the browser keeps the -// rule in one place for every future channel (a phone bridge has no -// AudioWorklet), and gives the server the signal it needs for barge-in: it has -// to know the user started talking while the assistant was still speaking. -// ============================================ -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import fs from 'node:fs/promises'; -import ort from 'onnxruntime-node'; -import logger from '../utils/logger.js'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); -const MODEL_URL = 'https://huggingface.co/onnx-community/silero-vad/resolve/main/onnx/model.onnx'; -const MODEL_PATH = path.join(process.env.SPEECH_CACHE_DIR || path.join(ROOT, '.transformers-cache'), 'silero-vad.onnx'); - -// Silero wants exactly 512 samples at 16 kHz (32 ms). The browser sends 40 ms -// chunks, so audio is buffered and drained in exact frames rather than forcing -// the client to match. -const FRAME = 512; -const RATE = 16000; -const FRAME_MS = (FRAME / RATE) * 1000; - -let sessionPromise = null; - -async function getSession() { - if (sessionPromise) return sessionPromise; - sessionPromise = (async () => { - try { - await fs.access(MODEL_PATH); - } catch { - logger.info('⬇️ fetching Silero VAD (2 MB)…'); - const res = await fetch(MODEL_URL); - if (!res.ok) throw new Error(`VAD download failed: ${res.status}`); - await fs.mkdir(path.dirname(MODEL_PATH), { recursive: true }); - await fs.writeFile(MODEL_PATH, Buffer.from(await res.arrayBuffer())); - } - const s = await ort.InferenceSession.create(MODEL_PATH); - logger.info('🎚️ Silero VAD ready'); - return s; - })().catch((e) => { sessionPromise = null; throw e; }); - return sessionPromise; -} - -export const vadOptions = { - threshold: Number(process.env.VAD_THRESHOLD ?? 0.5), - // Trailing silence that ends a turn. Too short truncates someone who pauses - // mid-sentence; too long makes the assistant feel sluggish. - silenceMs: Number(process.env.VAD_SILENCE_MS ?? 700), - // Ignore blips, so a cough or a door does not open a turn. - minSpeechMs: Number(process.env.VAD_MIN_SPEECH_MS ?? 250), - // Audio kept from BEFORE detection, so word onsets are not clipped. - prefixMs: Number(process.env.VAD_PREFIX_MS ?? 300), - maxUtteranceMs: Number(process.env.VAD_MAX_UTTERANCE_MS ?? 30000), -}; - -/** Streaming endpointer. One instance per connection. */ -export class Endpointer { - constructor(opts = {}) { - this.o = { ...vadOptions, ...opts }; - this.pending = new Float32Array(0); - this.prefixFrames = Math.max(1, Math.round(this.o.prefixMs / FRAME_MS)); - this.reset(); - } - - reset() { - this.speaking = false; - this.speechMs = 0; - this.silenceMs = 0; - this.buffer = []; - this.prefix = []; - // Silero is recurrent: this 2x1x128 state carries across frames and must - // be reset between turns or the model stays biased by the last utterance. - this.state = new ort.Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128]); - this.pending = new Float32Array(0); - } - - /** - * Feed float32 mono @16k. - * @returns {Promise<{utterances: Float32Array[], started: boolean}>} - * `started` flips the moment speech begins — that is the barge-in signal. - */ - async push(pcm) { - const session = await getSession(); - - const merged = new Float32Array(this.pending.length + pcm.length); - merged.set(this.pending); - merged.set(pcm, this.pending.length); - this.pending = merged; - - const utterances = []; - let started = false; - let offset = 0; - - while (this.pending.length - offset >= FRAME) { - const frame = this.pending.subarray(offset, offset + FRAME); - offset += FRAME; - - const out = await session.run({ - input: new ort.Tensor('float32', frame, [1, FRAME]), - sr: new ort.Tensor('int64', BigInt64Array.from([BigInt(RATE)]), []), - state: this.state, - }); - this.state = out.stateN ?? out.state_n ?? this.state; - const voiced = out.output.data[0] >= this.o.threshold; - - if (!this.speaking) { - this.prefix.push(Float32Array.from(frame)); - if (this.prefix.length > this.prefixFrames) this.prefix.shift(); - - if (voiced) { - this.speechMs += FRAME_MS; - if (this.speechMs >= this.o.minSpeechMs) { - this.speaking = true; - this.silenceMs = 0; - this.buffer = this.prefix; // open the turn with the pre-roll - this.prefix = []; - started = true; - } - } else { - this.speechMs = 0; - } - continue; - } - - this.buffer.push(Float32Array.from(frame)); - if (voiced) this.silenceMs = 0; - else this.silenceMs += FRAME_MS; - - const spokenMs = this.buffer.length * FRAME_MS; - if (this.silenceMs >= this.o.silenceMs || spokenMs >= this.o.maxUtteranceMs) { - utterances.push(concat(this.buffer)); - this.reset(); - } - } - - this.pending = this.pending.slice(offset); - return { utterances, started }; - } -} - -function concat(frames) { - const total = frames.reduce((n, f) => n + f.length, 0); - const out = new Float32Array(total); - let i = 0; - for (const f of frames) { out.set(f, i); i += f.length; } - return out; -} - -export const warmupVad = () => getSession().catch(() => {});