From aeaa72fc63fe0a911551db835bd631b89f281867 Mon Sep 17 00:00:00 2001 From: Bifang <915779419@qq.com> Date: Tue, 22 Sep 2026 14:35:42 +0800 Subject: [PATCH] first commit --- .env.example | 50 ++ .gitignore | 7 + README.md | 151 ++++ model_manifest.json | 84 ++ pyproject.toml | 20 + realtime_websocket/FIXES.md | 87 ++ realtime_websocket/README.md | 135 +++ realtime_websocket/auxiliary_service.py | 161 ++++ realtime_websocket/model_service.py | 120 +++ realtime_websocket/requirements.txt | 2 + realtime_websocket/server.py | 778 +++++++++++++++++ realtime_websocket/speaker_assembler.py | 186 ++++ realtime_websocket/static/app.js | 743 ++++++++++++++++ realtime_websocket/static/index.html | 119 +++ realtime_websocket/static/style.css | 805 ++++++++++++++++++ realtime_websocket/tests/test_frontend.cjs | 102 +++ .../tests/test_model_service.py | 32 + realtime_websocket/tests/test_server.py | 388 +++++++++ .../tests/test_speaker_assembler.py | 121 +++ requirements-auxiliary.txt | 6 + requirements-deploy.txt | 7 + scripts/__init__.py | 1 + scripts/auxiliary_server.py | 614 +++++++++++++ scripts/download_models.py | 230 +++++ scripts/model_manifest.py | 54 ++ scripts/serve.py | 216 +++++ tests/test_auxiliary_server.py | 169 ++++ tests/test_model_manifest.py | 73 ++ tests/test_serve.py | 43 + 29 files changed, 5504 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 model_manifest.json create mode 100644 pyproject.toml create mode 100644 realtime_websocket/FIXES.md create mode 100644 realtime_websocket/README.md create mode 100644 realtime_websocket/auxiliary_service.py create mode 100644 realtime_websocket/model_service.py create mode 100644 realtime_websocket/requirements.txt create mode 100644 realtime_websocket/server.py create mode 100644 realtime_websocket/speaker_assembler.py create mode 100644 realtime_websocket/static/app.js create mode 100644 realtime_websocket/static/index.html create mode 100644 realtime_websocket/static/style.css create mode 100644 realtime_websocket/tests/test_frontend.cjs create mode 100644 realtime_websocket/tests/test_model_service.py create mode 100644 realtime_websocket/tests/test_server.py create mode 100644 realtime_websocket/tests/test_speaker_assembler.py create mode 100644 requirements-auxiliary.txt create mode 100644 requirements-deploy.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/auxiliary_server.py create mode 100644 scripts/download_models.py create mode 100644 scripts/model_manifest.py create mode 100644 scripts/serve.py create mode 100644 tests/test_auxiliary_server.py create mode 100644 tests/test_model_manifest.py create mode 100644 tests/test_serve.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5901c32 --- /dev/null +++ b/.env.example @@ -0,0 +1,50 @@ +# 选择已经下载到 demo/models 目录中的 ASR 模型；必须与实际模型目录保持一致。 +# 默认使用轻量的 0.6B 模型。如下载的是 1.7B，请改为 1.7b，不能混用。 +QWEN3_ASR_MODEL=0.6b +# 本地模型根目录。下载脚本、VLLM 启动脚本和辅助服务都会从这里查找模型。 +MODEL_DIR=D:/github-project/ASR/Qwen-Asr/demo/models + +# VLLM 宿主机服务绑定地址；0.0.0.0 表示允许服务器网卡接收外部请求。 +VLLM_HOST=0.0.0.0 +# VLLM 服务端口；启动器、健康检查和前端连接地址必须使用同一个端口。 +VLLM_PORT=9950 +# VLLM 可执行文件名称；新版环境通常为 vllm，启动器会自动执行 vllm serve。 +VLLM_EXECUTABLE=vllm +# 启动成功提示中显示的地址，只影响日志和使用说明，不改变实际监听地址。 +VLLM_DISPLAY_HOST=127.0.0.1 +# 启动轮询使用的地址；如果 VLLM 部署在本机，通常保持 127.0.0.1 即可。 +VLLM_PROBE_HOST=127.0.0.1 +# VLLM 启动阶段最多检查多少次健康状态，超过次数仍未就绪则启动失败。 +VLLM_STARTUP_CHECK_LOOPS=60 +# 两次健康检查之间的等待秒数；模型加载较慢时可以适当增大检查次数。 +VLLM_STARTUP_CHECK_INTERVAL_SECONDS=2 +# VLLM 使用的显存比例。显存还要留给 VAD、聚类和声纹辅助模型时，建议预留余量。 +VLLM_GPU_MEMORY_UTILIZATION=0.3 +# VLLM 的最大上下文长度；数值越大通常占用越多显存，请结合显卡容量调整。 +VLLM_MAX_MODEL_LEN=16384 +# VLLM 同时处理的最大序列数；实时单路验证可保持默认值，多路并发时再调大。 +VLLM_MAX_NUM_SEQS=16 +# 张量并行 GPU 数量；单卡部署为 1，多卡部署时填写参与并行的 GPU 数量。 +VLLM_TENSOR_PARALLEL_SIZE=1 +# 是否启用 eager 模式。true 通常更容易启动和排查，false 可能获得更高性能。 +VLLM_ENFORCE_EAGER=true + +# GB10 需要使用 CUDA 13 工具链中的 ptxas；serve.py 会自动加载该配置并传给 VLLM。 +TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas + +# 可选：为外部客户端设置稳定的公开模型名称。留空时默认使用完整 ModelScope ID。 +# VLLM_SERVED_MODEL_NAME=Qwen/Qwen3-ASR-0.6B + +# 辅助模型服务地址；它独立加载 VAD、CAM++ 聚类和声纹模型，WebSocket +# 服务只通过 HTTP 调用，不会把这些 GPU 模型重复加载到 WebSocket 进程。 +AUXILIARY_SERVICE_URL=http://127.0.0.1:8010 +# WebSocket 调用已独立部署的 vLLM；跨服务器时改成模型服务器的实际地址。 +MODEL_SERVICE_URL=http://127.0.0.1:9950/v1 +# 辅助服务监听的 GPU；单卡服务器保持 cuda:0，多卡时可改成指定卡号。 +AUXILIARY_DEVICE=cuda:0 +# 仅保留兼容旧配置；VAD/CAM++ 核心模型缺失时始终拒绝启动，不会静默降级。 +# 可选 diarization/aligner 缺失不会阻断实时服务。 +AUXILIARY_ALLOW_MISSING=false +# 启动时强制加载 VAD + CAM++ speaker_verification；完整 diarization 首次调用时按需加载。 +# 如确实需要启动时额外预加载，可追加：vad,speaker_verification,diarization +AUXILIARY_PRELOAD_KINDS=vad,speaker_verification diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..23c14f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.py[cod] + +.code-review-graph/ +models/ +.env \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0140208 --- /dev/null +++ b/README.md @@ -0,0 +1,151 @@ +# Qwen3-ASR VLLM 独立部署项目 + +本目录是后续实时 ASR 功能验证使用的独立模型服务项目。 + +它不导入、不启动、也不调用仓库根目录下原项目的 `app/` 代码。模型下载、VLLM 启动、配置和服务验证都在本目录内完成。后续验证 demo 只需要调用这里提供的 VLLM OpenAI 兼容接口。 + +## 当前下载范围 + +默认下载一个 ASR 模型和独立辅助模型运行服务所需的全部模型资产： + +```text +Qwen/Qwen3-ASR-0.6B +``` + +ASR 如需使用大模型，可显式选择 1.7B： + +```text +Qwen/Qwen3-ASR-1.7B +``` + +辅助资产包括 VAD、CAM++ 分离、配置声纹、实时声纹、CAM++ Transformer 和 Qwen3 ForcedAligner。它们不会由 `qwen-asr-serve` 启动，而是由独立 Python 辅助模型服务加载。 + +不会下载另一个未选择的 ASR 模型；辅助模型资产会随默认部署包下载，供独立 Python 运行服务预加载。 + +## 1. 下载模型 + +模型直接下载到宿主机的 `demo/models`。ASR 由 VLLM 启动，辅助模型由独立 Python 运行服务启动。 + +```powershell +cd D:\github-project\ASR\Qwen-Asr\demo +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -r requirements-download.txt +python scripts\download_models.py +``` + +上述命令会下载默认 `0.6B` ASR 以及全部辅助模型，并在下载完成后把 CAM++ 配置中的依赖模型 ID 改为 `demo/models` 下的本地路径，保证辅助服务可以离线启动。只下载 ASR 时使用： + +```powershell +python scripts\download_models.py --skip-auxiliary +``` + +只下载辅助模型时使用： + +```powershell +python scripts\download_models.py --auxiliary-only +``` + +选择 1.7B： + +```powershell +python scripts\download_models.py --model 1.7b +``` + +检查模型是否完整但不下载： + +```powershell +python scripts\download_models.py --check-only +``` + +ModelScope 下载也可以通过环境变量调整缓存目录： + +```powershell +$env:MODELSCOPE_CACHE = 'D:\modelscope-cache' +python scripts\download_models.py +``` + +## 2. 宿主机启动 VLLM 服务 + +需要宿主机具备与 VLLM 兼容的 Python、CUDA 和 NVIDIA 驱动环境。安装部署依赖： + +```bash +python -m pip install -r requirements-deploy.txt +``` + +先复制并按服务器实际路径修改 `.env`，启动器会自动读取该文件： + +```bash +cp .env.example .env +``` + +默认启动 `Qwen/Qwen3-ASR-0.6B`，监听地址为 `0.0.0.0:9950`： + +```bash +python scripts/serve.py +``` + +模型和启动检查循环可以通过命令行或环境变量传入；端口统一在 `scripts/serve.py` 的 `SERVER_PORT` 变量中维护： + +```bash +QWEN3_ASR_MODEL=0.6b VLLM_STARTUP_CHECK_LOOPS=120 \ +VLLM_STARTUP_CHECK_INTERVAL_SECONDS=2 python scripts/serve.py +``` + +如果使用 `1.7B`，下载和启动必须指定同一个模型： + +```powershell +python scripts\download_models.py --model 1.7b +python -m scripts.serve --model 1.7b +``` + +启动器会按 `VLLM_STARTUP_CHECK_LOOPS` 次数轮询 `/health`，每次间隔由 `VLLM_STARTUP_CHECK_INTERVAL_SECONDS` 指定。服务端口、健康检查端口和就绪提示统一使用 `scripts/serve.py` 中的 `SERVER_PORT`。 + +服务启动后可检查： + +```bash +curl "http://${VLLM_DISPLAY_HOST:-127.0.0.1}:9950/health" +curl "http://${VLLM_DISPLAY_HOST:-127.0.0.1}:9950/v1/models" +``` + +## 3. 调用转写接口 + +VLLM 服务提供 OpenAI 兼容的音频转写接口： + +```bash +curl "http://${VLLM_DISPLAY_HOST:-127.0.0.1}:9950/v1/audio/transcriptions" \ + -H "Authorization: Bearer EMPTY" \ + -F "file=@./audio/sample.wav" \ + -F "model=Qwen/Qwen3-ASR-0.6B" +``` + +## 4. 启动辅助模型服务 + +另开一个终端，在同一台服务器启动 VAD、CAM++ 和声纹模型运行服务： + +```powershell +cd D:\github-project\ASR\Qwen-Asr\demo +pip install -r requirements-auxiliary.txt +python scripts\auxiliary_server.py +``` + +辅助服务默认监听 `0.0.0.0:8010`。实时链路启动时严格加载 VAD 和 CAM++ `speaker_verification` 声纹模型，用于每个 turn 的特征提取与在线聚类；完整 CAM++ 分离、Transformer 和 ForcedAligner 不阻断核心服务，完整分离模型会在调用 `/v1/diarization` 时按需加载。检查状态： + +```bash +curl http://127.0.0.1:8010/health +``` + +WebSocket demo 默认连接 `9950` 的 ASR VLLM，辅助服务使用 `8010`。`/health` 的 `ready` 要求 `vad_ready` 与 `speaker_embedding_ready` 同时为 true；完整 diarization 资产缺失不会影响实时 `/v1/speaker/resolve`。 + +实时链路中的职责是：WebSocket 用 RMS 帧门控快速检测停顿；辅助服务用 FunASR VAD 提供 `/v1/vad`，并加载 CAM++ `speaker_verification` 提取 turn embedding，再由服务端在线聚类。`speech_campplus_speaker-diarization_common` 是完整音频分离接口的额外 pipeline，不是实时 turn 聚类的唯一入口。 + +也可以使用多模态 Chat Completions 接口，后续实时验证项目将以此服务边界为准。 + +## 项目边界 + +- `scripts/download_models.py`：下载选定 ASR 和全部辅助模型资产。 +- `scripts/serve.py`：读取 `.env`，解析模型选择、宿主机参数并启动新版 `vllm serve`。 +- `requirements-deploy.txt`：安装宿主机部署所需的官方 Qwen3-ASR VLLM 依赖。 +- `tests/`：只验证本项目自己的模型清单和选择逻辑，不依赖原项目。 + +模型服务就绪后，新的实时 ASR demo 放在同级 `demo` 项目中继续开发，但不得通过 Python import 或 HTTP/WebSocket 调用原项目服务。 diff --git a/model_manifest.json b/model_manifest.json new file mode 100644 index 0000000..a96f58e --- /dev/null +++ b/model_manifest.json @@ -0,0 +1,84 @@ +{ + "default_model": "Qwen/Qwen3-ASR-0.6B", + "models": { + "Qwen/Qwen3-ASR-1.7B": { + "alias": "1.7b", + "directory": "Qwen/Qwen3-ASR-1.7B", + "description": "Qwen3-ASR 1.7B，GPU 可选大模型" + }, + "Qwen/Qwen3-ASR-0.6B": { + "alias": "0.6b", + "directory": "Qwen/Qwen3-ASR-0.6B", + "description": "Qwen3-ASR 0.6B，GPU 默认轻量模型" + } + }, + "auxiliary_models": { + "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch": { + "alias": "vad", + "directory": "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", + "kind": "vad", + "description": "FunASR FSMN VAD", + "revision": "v2.0.2", + "required_files": ["configuration.json", "config.yaml", "model.pb"], + "min_total_size_bytes": 1000000 + }, + "iic/speech_campplus_speaker-diarization_common": { + "alias": "diarization", + "directory": "iic/speech_campplus_speaker-diarization_common", + "kind": "diarization", + "description": "CAM++ speaker diarization", + "required_files": [ + "configuration.json", + "config.yaml", + "onnx/asd.onnx", + "onnx/face_recog_ir101.onnx", + "onnx/fqa.onnx", + "onnx/version-RFB-320.onnx" + ], + "min_total_size_bytes": 50000000 + }, + "iic/speech_campplus_sv_zh-cn_16k-common": { + "alias": "speaker", + "directory": "iic/speech_campplus_sv_zh-cn_16k-common", + "kind": "speaker_verification", + "description": "Configured CAM++ speaker verification", + "revision": "v2.0.2", + "required_files": ["configuration.json", "config.yaml", "campplus_cn_common.bin"], + "min_total_size_bytes": 10000000 + }, + "iic/speech_eres2netv2_sv_zh-cn_16k-common": { + "alias": "realtime-speaker", + "directory": "iic/speech_eres2netv2_sv_zh-cn_16k-common", + "kind": "realtime_speaker_verification", + "description": "Realtime speaker verification", + "required_files": ["configuration.json"], + "any_files": ["*"], + "min_total_size_bytes": 10000000 + }, + "damo/speech_campplus_sv_zh-cn_16k-common": { + "alias": "speaker-damo", + "directory": "damo/speech_campplus_sv_zh-cn_16k-common", + "kind": "speaker_verification", + "description": "CAM++ speaker verification dependency", + "required_files": ["configuration.json", "config.yaml", "campplus_cn_common.bin"], + "min_total_size_bytes": 10000000 + }, + "damo/speech_campplus-transformer_scl_zh-cn_16k-common": { + "alias": "speaker-transformer", + "directory": "damo/speech_campplus-transformer_scl_zh-cn_16k-common", + "kind": "speaker_transformer", + "description": "CAM++ Transformer dependency", + "required_files": ["configuration.json", "campplus_cn_encoder.pt", "transformer_backend.pt"], + "min_total_size_bytes": 10000000 + }, + "Qwen/Qwen3-ForcedAligner-0.6B": { + "alias": "forced-aligner", + "directory": "Qwen/Qwen3-ForcedAligner-0.6B", + "kind": "forced_aligner", + "description": "Qwen3 word-level forced aligner", + "required_files": ["config.json"], + "any_files": ["*.safetensors", "*.bin"], + "min_total_size_bytes": 500000000 + } + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..30ed923 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "qwen3-asr-vllm-deployment" +version = "0.1.0" +description = "Standalone Qwen3-ASR model downloader and VLLM deployment" +requires-python = ">=3.10,<3.14" +dependencies = [ + "modelscope==1.34.0", + "qwen-asr[vllm]==0.0.6", +] + +[project.scripts] +qwen3-asr-download = "scripts.download_models:main" +qwen3-asr-serve = "scripts.serve:main" + +[tool.setuptools] +packages = ["scripts"] diff --git a/realtime_websocket/FIXES.md b/realtime_websocket/FIXES.md new file mode 100644 index 0000000..f2d61d6 --- /dev/null +++ b/realtime_websocket/FIXES.md @@ -0,0 +1,87 @@ +# Demo 说话人链路修复与验收 + +本次仅修改 `demo/`。参考原 `app/services/qwen3_websocket_asr.py` 的前滚音频、同句异步说话人更新和结束提交方式，保留独立 vLLM ASR + 辅助声纹服务 + WebSocket 三个进程。当前机器没有显卡，验证采用模拟模型，真实声纹准确率和模型加载状态需在部署机器上验收。 + +## 已确认的代码问题 + +1. 页面只处理 `sentences`，忽略已实现的 `display_state`，因此后端合并结果没有展示。未知片段还会临时附在上一位已确认说话人的气泡里。 +2. 页面点击停止后 5 秒强制断开，但辅助 HTTP 请求的超时是 45 秒，可能截掉迟到的声纹更新。 +3. 无 embedding、低置信度、协议字段不完整等情况被静默丢弃，用户无法区分等待、短音频、服务错误和证据拒绝。 +4. 旧声纹长度检查把句尾 800ms 静音一起计算，可能让短插话通过长度要求；WAV 固定去掉 44 字节也会破坏带额外元数据的输入。 +5. 仅 vLLM 启动器加载 `demo/.env`，另两个进程没有加载；`0.6b` 下载别名也可能被直接当作公开模型名发送。 + +这些是代码中可复现的问题，不能据此断言部署中的声纹模型一定已经正常加载。新版本将具体原因直接显示在片段气泡和日志里。 + +## 修复后的行为 + +- partial/final 和声纹更新继续使用同一 `sentence_id`。页面显示完整聚合快照，过时的 `revision` 不覆盖新快照。 +- 仅合并相邻且身份可信的片段。未知短插话独立显示；A→B→A 保留顺序。不同实名或实名与弱匿名身份不会因为簇 ID 相同而合并。 +- 每条声纹证据必须来自当前片段；同步、异步两种入口都拒绝 `short_attach` / `embedding_attach`。不复制上一段 embedding。声纹向量拒绝零向量、NaN、Infinity 和多样本矩阵。 +- WebSocket 以有效有声帧判断长度。少于 800ms 的语音保持 pending；800ms～1.6s 也必须独立提取特征，不能直接继承前一位身份。该长度门槛属于保守保护，不能保证短样本识别准确率。 +- 保留 200ms 前滚，提交时去除尾部静音。增量解析 WAV 的 RIFF/fmt/data/JUNK 等头，拒绝非 16kHz 单声道 PCM16 文件。 +- `stop/eof` 返回 `draining`，排空 ASR 和声纹队列后才发送最终快照及 `end`。声纹失败不阻断 ASR。abort、断线、异常和正常结束均清理聚类会话。 +- ASR worker 异常会立即发送 `error`，不会一直等待客户端停止。 + +## 如何判断卡在哪里 + +页面顶部显示已确认数量；未匹配到说话人的气泡统一显示“未知说话人”，详细等待或失败原因通过标签悬停提示和原始日志查看。原始日志保留 `speaker_status`、`speaker_reason`、`speaker_strategy` 和置信度。 + +| `speaker_status` | 含义与排查方向 | +|---|---| +| `waiting_final` | 讲话仍在进行，等待静音或最大时长切段 | +| `queued` / `processing` | 文本已完成，声纹正在排队或推理 | +| `confirmed` | 当前片段声纹已确认，应该显示说话人标签 | +| `insufficient_audio` | 有效语音过短，不继承上一位；使用较长发言复测 | +| `service_unavailable` / `service_error` | 未配置、无法访问或模型推理失败；检查辅助服务及完整错误 | +| `no_embedding` | 服务没有产生可用特征 | +| `evidence_rejected` | 缺少 fresh/confirmed 证据、置信度不足或使用了继承策略 | +| `disabled` | 本次未开启说话人分离 | + +健康检查为 `http://辅助服务器:8010/health`。新版本有 `speaker_protocol_version: 2`，重点检查 `speaker_embedding_ready` 与 `speaker_embedding_model`。若没有版本字段，检查是否重启了更新后的辅助进程。HTTP 健康检查不执行真实声纹推理，不能代替音频验收。 + +辅助服务目前返回匿名的“说话人 1、2……”；demo 没有接入原应用的声纹注册库，因此不会自动识别人员实名。vLLM 仅输出转写文本，声纹标签由 `scripts/auxiliary_server.py` 负责。 + +模型职责要区分：`iic/speech_campplus_sv_zh-cn_16k-common` 是实时 turn 的 CAM++ embedding 模型，必须加载；`iic/speech_campplus_speaker-diarization_common` 是完整音频分离 pipeline，包含额外的 change locator/VAD 依赖，当前实时 WebSocket 不在启动阶段调用它。WebSocket 自身仍用轻量 RMS 帧门控切句，辅助服务的 FunASR VAD 对外提供 `/v1/vad`，并供完整 diarization 依赖使用；因此启动辅助服务是为了 CAM++ 声纹和聚类，不能把整段 diarization 的加载失败误认为 vLLM 失败。 + +## 部署后操作 + +在部署机更新这些文件后，已有 vLLM 可继续运行。重新安装增补的 `python-dotenv` 依赖，并重启辅助服务及 WebSocket。辅助服务启动时强制依赖 VAD + CAM++ `speaker_verification`；完整 CAM++ diarization 不再阻断实时启动；以下命令都从 `demo/` 目录执行： + +```text +pip install -r requirements-auxiliary.txt +pip install -r realtime_asr_optimization_demo/requirements.txt +python scripts/auxiliary_server.py +``` + +如果仍提示核心模型缺失或加载失败，日志会列出模型 ID、实际查找路径、状态和底层异常。先执行 `python scripts/download_models.py --auxiliary-only`，或设置 `.env` 的 `MODEL_DIR` 指向同时包含 `damo/speech_fsmn_vad_zh-cn-16k-common-pytorch` 与 `iic/speech_campplus_sv_zh-cn_16k-common` 的目录。不要用 `AUXILIARY_ALLOW_MISSING=true` 掩盖 VAD/CAM++ 核心模型缺失；该选项只适合临时查看可选模型状态。 + +另一个终端执行： + +```text +python realtime_asr_optimization_demo/server.py --no-browser +``` + +三个进程现在均读取 `demo/.env`，系统环境变量优先。确认 `MODEL_SERVICE_URL` 指向现有 vLLM 的 `/v1`，`AUXILIARY_SERVICE_URL` 指向辅助服务。`QWEN3_ASR_MODEL` 支持部署清单别名，设置 `VLLM_SERVED_MODEL_NAME` 时优先使用该公开名称。更新后刷新浏览器；脚本 URL 已更新版本号。 + +`127.0.0.1` 指各 Python 服务运行的机器，不是浏览器所在机器。跨服务器部署时填写对应服务器 IP。浏览器麦克风访问远程页面需要安全上下文（HTTPS）；本机 localhost 可用于测试。 + +## 验收顺序 + +1. 单人讲话 2～4 秒后停顿：先出现文本，随后同句变为“说话人 1”。 +2. 同一人再次讲话并停顿：确认后相邻块应合并；取消页面“合并相邻”可对照物理片段。 +3. A→B→A，各说 2 秒以上：应保持三个时间顺序块。标签准确率需要真实声纹模型验证。 +4. A 后 B 说一个很短的“嗯”：应显示“有效语音不足”，不能进入 A 的气泡。 +5. 辅助服务关闭时测试：ASR 仍完成，片段明确显示服务错误。 +6. 讲话中点击停止：等待最终结果，不能在五秒时丢失说话人更新。 + +无显卡回归命令： + +```text +cd demo +python -m unittest discover -s tests -v +cd realtime_asr_optimization_demo +python -m unittest discover -s tests -v +node --test tests/test_frontend.cjs +``` + +这些测试覆盖状态机、模拟 HTTP/WebSocket、延迟更新、前端脚本和有效向量校验，不执行模型下载或 GPU 推理。当前 VLLM 适配器仍是 HTTP 累积窗口 partial，`native_partial_supported=false`；本次没有把 HTTP 接口包装成原生增量模型状态。单个内部片段中无停顿的多人换话或重叠讲话仍需真实模型和更细粒度切段验证。 diff --git a/realtime_websocket/README.md b/realtime_websocket/README.md new file mode 100644 index 0000000..479647d --- /dev/null +++ b/realtime_websocket/README.md @@ -0,0 +1,135 @@ +# Realtime ASR WebSocket Optimization Demo + +这是一个独立的实时 ASR WebSocket 验证项目，放在模型部署项目 `demo` 下，但不导入、不启动、也不调用仓库根目录的原始 `app/`。 + +## 验证目标 + +- 浏览器麦克风或按实时速度发送的 PCM/WAV 音频通过本项目 WebSocket 发送。 +- 同一 `sentence_id` 的 partial、final 和后续更新覆盖同一条 raw segment。 +- raw segment 与前端 display block 分离，避免把物理切段直接等同于展示换行。 +- 小于 1.6 秒且带有 `short_attach` / `embedding_attach` 策略的实名结果降级为 pending。 +- 不把 embedding 字段写入 demo 状态池。 +- 只有相邻且身份可信的 segment 才合并；A→B→A 保持时间顺序。 +- 记录 partial 首次延迟、final 延迟、partial 修订次数和服务端返回时间范围。 +- 每个实时 turn 单独提交声纹特征，由辅助服务维护本 WebSocket session 的在线聚类中心。 + +本次修复、诊断状态和部署验收步骤见 [FIXES.md](FIXES.md)。ASR 继续使用已部署的独立 vLLM；已有 vLLM 服务时无需重复启动或下载模型。 + +当前 VLLM 端点提供的是同步 OpenAI 音频转写接口，没有暴露原生 +`create_stream/feed_stream/finish_stream`。因此本项目仍然是真实 WebSocket +音频流：麦克风 PCM 到达后立即进入 VAD，按窗口调用 VLLM 生成 partial；它不会 +等整段音频结束。`native_partial_supported=false` 只表示模型 HTTP 接口本身 +不是原生 ASR stream，不伪造不存在的能力。 + +## 启动 + +需要启动两个模型服务和一个 WebSocket 页面：VLLM 只负责 ASR，辅助服务负责 +VAD、CAM++ 声纹模型及在线聚类，WebSocket 只做音频流编排，不导入原项目代码。 + +先在 `demo` 目录下载 ASR 和辅助模型，并启动 VLLM： + +```powershell +cd D:\github-project\ASR\Qwen-Asr\demo +python scripts\download_models.py +python scripts\serve.py +``` + +另一个终端启动辅助模型服务： + +```powershell +cd D:\github-project\ASR\Qwen-Asr\demo +pip install -r requirements-auxiliary.txt +python scripts\auxiliary_server.py +``` + +另开一个终端启动 WebSocket 页面： + +```powershell +cd D:\github-project\ASR\Qwen-Asr\demo\realtime_asr_optimization_demo +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -r requirements.txt +python server.py --no-browser +``` + +页面服务默认监听 `0.0.0.0:8082`，端口在 `server.py` 顶部的 `WEB_PORT` 内部变量中维护。 +VLLM 默认地址为 `http://127.0.0.1:9950/v1`，辅助服务默认地址为 +`http://127.0.0.1:8010`。可通过环境变量切换到远程服务： + +```powershell +$env:MODEL_SERVICE_URL = 'http://127.0.0.1:9950/v1' +$env:AUXILIARY_SERVICE_URL = 'http://127.0.0.1:8010' +python server.py --no-browser +``` + +如果服务器端口已通过 VS Code Remote/端口转发映射到本机，保持上述两个 +`127.0.0.1` 地址即可：本地 WebSocket 只负责编排，ASR 和 VAD/CAM++ 推理仍在 +服务器 GPU 服务中完成。先访问 `http://127.0.0.1:9950/v1/models` 与 +`http://127.0.0.1:8010/health`，分别确认 VLLM 模型和辅助模型服务可达且 `ready=true`。 + +服务器部署时使用 `--no-browser`，然后在客户端浏览器访问 `http://服务器IP:8082`。如需让启动日志显示服务器域名或 IP，可设置 `WEB_DISPLAY_HOST`；它只影响提示文本，不改变监听地址。 + +辅助服务启动后可用 `http://服务器IP:8010/health` 检查模型状态。WebSocket +收到聚类服务错误时仍会继续输出 ASR，但对应片段会显示“未知说话人”；详细的 +`speaker_reason` 可将鼠标悬停在标签上查看，事件日志仍会显示 `speaker_warning`， +便于区分“模型未归类”和“ASR 失败”。 + +声纹服务的实时路径必须通过 ModelScope pipeline 的公开接口提取 embedding： +`pipeline([wav_path], output_emb=True)`。不能绕过 pipeline 预处理后直接调用 +`pipeline.model`，否则采样率、声道和 waveform 预处理不会执行，部分 ModelScope +版本会直接抛异常，WebSocket 仍会继续输出 ASR 并把说话人保留为 pending。 +更新辅助服务代码后需要重启 `python scripts/auxiliary_server.py`，仅重启页面 +服务不会替换已经驻留在 GPU 中的旧辅助服务进程。 + +## 页面操作 + +1. 选择 Mic 或 File。 +2. 点击开始，浏览器通过 `/ws` 建立本项目 WebSocket。 +3. 页面展示腾讯 Demo 风格的气泡；麦克风或 PCM/WAV 文件按流式方式输入，partial 会在讲话过程中实时刷新。 +4. VAD 检测到静音后提交当前 turn，先返回 pending，再异步更新说话人。 +5. 停止会发送 `stop`，服务端完成当前 turn 和 speaker 队列后再发送 `end`。 +6. `abort` 只取消会话，不提交当前片段。 + +## WebSocket 消息 + +客户端首条消息： + +```json +{ + "type": "start", + "source": "mic", + "model_service_url": "http://127.0.0.1:9950/v1", + "model": "Qwen/Qwen3-ASR-0.6B", + "speaker_diarization": 1, + "sentence_strategy": 0, + "enable_native_partial_stream": true, + "partial_interval_ms": 1200, + "max_segment_sec": 12, + "speaker_gap_ms": 400, + "display_merge": true +} +``` + +随后持续发送 16kHz、单声道、PCM16 二进制音频；文件模式仅支持 PCM/WAV，结束发送 `{"type":"eof"}`，停止发送 +`{"type":"stop"}`，取消发送 `{"type":"abort"}`。每个已完成 turn 会向辅助服务 +发送一次 `/v1/speaker/resolve`，只包含当前 turn 音频和 session_id，不会重复上传整段会话。 + +服务端会发送 `start`、`sentences`、`display_state`、`metrics`、`speaker_warning`、`draining`、`end` 和 `error`。 +页面用带 `revision` 的 `display_state` 渲染，以 `block_id` 标识展示块；`sentences` 保留原始片段及诊断状态。 +停止后必须等待 `end`，其中包含完整 `sentences` 和 `display_blocks`，不能提前关闭连接。 +`sentences` 中 `sentence_type=0` 是 partial，`sentence_type=1` 是 final；同一个 +`sentence_id` 必须覆盖更新而不是追加。`sentence_strategy=0` 使用约 800ms +静音切句，`sentence_strategy=1` 使用约 1400ms 静音切段，更适合段落模式。 + +说话人模式下，`speaker_gap_ms`（默认 400ms）会在有效语音达到 800ms 后 +提前结束短停顿话轮，用于拆开 A→B 的交接；它不触发额外声纹请求，同一说话人的 +相邻片段仍由辅助服务聚类为同一说话人。服务未就绪时该策略会自动关闭。 + +## 目录 + +- `server.py`：本地 HTTP 页面和 WebSocket 会话编排。 +- `model_service.py`：独立 VLLM OpenAI 音频接口适配层。 +- `auxiliary_service.py`：独立辅助模型 HTTP 接口适配层。 +- `speaker_assembler.py`：raw segment、speaker evidence 和 display block 状态机。 +- `static/`：麦克风/文件测试页面。 +- `tests/`：只测试本项目状态合并和音频转换逻辑。 diff --git a/realtime_websocket/auxiliary_service.py b/realtime_websocket/auxiliary_service.py new file mode 100644 index 0000000..d67aadf --- /dev/null +++ b/realtime_websocket/auxiliary_service.py @@ -0,0 +1,161 @@ +"""独立 WebSocket Demo 使用的 VAD 和说话人辅助服务客户端。""" + +from __future__ import annotations + +import io +import wave +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from aiohttp import ClientSession, ClientTimeout, FormData + + +@dataclass(frozen=True) +class AuxiliaryServiceConfig: + """辅助模型服务的 HTTP 连接配置。""" + + base_url: str = "http://127.0.0.1:8010" + timeout_seconds: float = 45.0 + + +def pcm16_to_wav(pcm_bytes: bytes, sample_rate: int = 16000) -> bytes: + """将 Demo 内部的 16kHz 单声道 PCM16 封装成辅助服务可读取的 WAV。""" + output = io.BytesIO() + with wave.open(output, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(sample_rate) + wav_file.writeframes(pcm_bytes) + return output.getvalue() + + +class AuxiliaryModelService: + """调用独立辅助模型服务，不在 WebSocket 进程内加载 GPU 模型。""" + + def __init__(self, config: AuxiliaryServiceConfig) -> None: + self.config = config + self._session: ClientSession | None = None + + async def start(self) -> None: + """创建可复用的 HTTP 会话，避免每个片段重复建立 TCP 连接。""" + self._session = ClientSession(timeout=ClientTimeout(total=self.config.timeout_seconds)) + + async def close(self) -> None: + """关闭辅助服务 HTTP 会话。""" + if self._session is not None: + await self._session.close() + self._session = None + + async def health(self) -> dict[str, Any]: + """读取辅助服务健康状态，避免服务不可达时只能看到 ASR 的降级结果。""" + if self._session is None: + raise RuntimeError("auxiliary model service is not started") + endpoint = self.config.base_url.rstrip("/") + "/health" + async with self._session.get(endpoint) as response: + body = await response.text() + if response.status >= 400: + raise RuntimeError(f"auxiliary health check failed ({response.status}): {body[:500]}") + try: + decoded = await response.json(content_type=None) + except ValueError as exc: + raise RuntimeError(f"auxiliary health check returned invalid JSON: {body[:500]}") from exc + if not isinstance(decoded, dict): + raise RuntimeError("auxiliary health check returned a non-object JSON value") + return decoded + + async def resolve_speaker( + self, + pcm_bytes: bytes, + session_id: str, + start_time_ms: float, + end_time_ms: float, + ) -> dict[str, Any] | None: + """提交一个已经由实时 VAD 完成的 turn，获取在线聚类结果。 + + 每次请求只包含当前 turn，不上传整段会话；辅助服务通过 session_id + 保存聚类中心，因此同一说话人在 A→B→A 场景下仍能保持同一标签。 + """ + if self._session is None: + raise RuntimeError("auxiliary model service is not started") + form = FormData() + form.add_field("file", pcm16_to_wav(pcm_bytes), filename="turn.wav", content_type="audio/wav") + form.add_field("session_id", session_id) + form.add_field("start_time_ms", str(start_time_ms)) + form.add_field("end_time_ms", str(end_time_ms)) + endpoint = self.config.base_url.rstrip("/") + "/v1/speaker/resolve" + async with self._session.post(endpoint, data=form) as response: + body = await response.text() + if response.status >= 400: + raise RuntimeError(f"auxiliary speaker resolve failed ({response.status}): {body[:500]}") + try: + decoded = await response.json(content_type=None) + except ValueError as exc: + raise RuntimeError(f"auxiliary speaker resolve returned invalid JSON: {body[:500]}") from exc + if not isinstance(decoded, dict): + raise RuntimeError("auxiliary speaker resolve returned a non-object JSON value") + if decoded.get("error"): + raise RuntimeError(str(decoded["error"])) + # 保留无标签响应里的具体原因；由组装器统一判断可信度，避免这里静默丢弃。 + return decoded + + async def reset_speaker_session(self, session_id: str) -> None: + """通知辅助服务释放当前 WebSocket 对应的在线聚类状态。""" + if self._session is None: + return + endpoint = self.config.base_url.rstrip("/") + "/v1/speaker/reset" + try: + async with self._session.post(endpoint, json={"session_id": session_id}) as response: + await response.read() + except Exception: + # 清理失败不能影响已经完成的 ASR 结果，辅助服务会自行过期清理。 + return + + async def diarize( + self, + audio_bytes: bytes, + source: str = "mic", + file_name: str = "audio.wav", + ) -> list[dict[str, Any]]: + """提交完整会话音频，返回带毫秒时间范围和标签的聚类片段。 + + 麦克风、PCM 和 WAV 在 WebSocket 层已经能被识别为 16kHz PCM； + MP3、M4A 等压缩文件必须保留原始容器，否则把压缩字节直接包装成 + PCM 会得到不可用的声纹输入。 + """ + if self._session is None: + raise RuntimeError("auxiliary model service is not started") + suffix = Path(file_name).suffix.lower() + is_pcm = source == "mic" or suffix == ".pcm" + if is_pcm: + payload = pcm16_to_wav(audio_bytes) + upload_name = "session.wav" + content_type = "audio/wav" + elif suffix == ".wav": + payload = audio_bytes + upload_name = "session.wav" + content_type = "audio/wav" + else: + payload = audio_bytes + upload_name = Path(file_name).name or "session.audio" + content_type = { + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + }.get(suffix, "application/octet-stream") + form = FormData() + form.add_field("file", payload, filename=upload_name, content_type=content_type) + endpoint = self.config.base_url.rstrip("/") + "/v1/diarization" + async with self._session.post(endpoint, data=form) as response: + body = await response.text() + if response.status >= 400: + raise RuntimeError(f"auxiliary diarization failed ({response.status}): {body[:500]}") + try: + decoded = await response.json(content_type=None) + except ValueError as exc: + raise RuntimeError(f"auxiliary diarization returned invalid JSON: {body[:500]}") from exc + raw_segments = decoded.get("segments", []) if isinstance(decoded, dict) else [] + if not isinstance(raw_segments, list): + return [] + return [segment for segment in raw_segments if isinstance(segment, dict)] diff --git a/realtime_websocket/model_service.py b/realtime_websocket/model_service.py new file mode 100644 index 0000000..cc0288d --- /dev/null +++ b/realtime_websocket/model_service.py @@ -0,0 +1,120 @@ +"""独立实时 Demo 使用的 OpenAI 兼容 VLLM 服务适配器。""" + +from __future__ import annotations + +import io +import wave +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from aiohttp import ClientSession, ClientTimeout, FormData + + +@dataclass(frozen=True) +class ModelServiceConfig: + """一个独立 VLLM 端点所需的连接配置。""" + + base_url: str = "http://127.0.0.1:9950/v1" + model: str = "Qwen/Qwen3-ASR-0.6B" + api_key: str = "EMPTY" + timeout_seconds: float = 45.0 + + +def pcm16_to_wav(pcm_bytes: bytes, sample_rate: int = 16000) -> bytes: + """将浏览器发送的 PCM16 单声道数据封装为 VLLM 可识别的 WAV 请求。""" + output = io.BytesIO() + with wave.open(output, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(sample_rate) + wav_file.writeframes(pcm_bytes) + return output.getvalue() + + +def wav_to_pcm16(audio_bytes: bytes) -> bytes: + """从 WAV 缓冲区提取 PCM 帧，并兼容尚未完整的中间音频数据。""" + try: + with wave.open(io.BytesIO(audio_bytes), "rb") as wav_file: + return wav_file.readframes(wav_file.getnframes()) + except (EOFError, wave.Error): + if audio_bytes[:4] == b"RIFF" and audio_bytes[8:12] == b"WAVE" and len(audio_bytes) > 44: + return audio_bytes[44:] + return audio_bytes + + +def prepare_audio_request( + audio_bytes: bytes, + source: str, + file_name: str, + partial: bool, +) -> tuple[bytes, str, str] | None: + """将麦克风、PCM 或 WAV 数据转换为 WAV；压缩格式的中间片段延迟到最终帧处理。""" + suffix = Path(file_name).suffix.lower() + if source == "mic" or suffix in {".pcm", ".wav"}: + pcm_bytes = wav_to_pcm16(audio_bytes) if suffix == ".wav" else audio_bytes + return pcm16_to_wav(pcm_bytes), "audio.wav", "audio/wav" + if partial: + # MP3/M4A/OGG 的不断增长前缀通常不是完整容器，不能安全解码，因此只在 + # 最终阶段提交压缩文件，避免中间请求产生随机解码错误。 + return None + content_type = { + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + }.get(suffix, "application/octet-stream") + return audio_bytes, Path(file_name).name or "audio.bin", content_type + + +class VLLMTranscriptionService: + """只调用独立项目提供的 VLLM HTTP 接口，不导入原项目应用代码。""" + + native_partial_supported = False + + def __init__(self, config: ModelServiceConfig) -> None: + self.config = config + self._session: ClientSession | None = None + + async def start(self) -> None: + """创建可复用的 HTTP 会话，供所有中间和最终转写请求共享。""" + self._session = ClientSession(timeout=ClientTimeout(total=self.config.timeout_seconds)) + + async def close(self) -> None: + """本地 Demo 退出时释放可复用的 HTTP 会话和底层连接。""" + if self._session is not None: + await self._session.close() + self._session = None + + async def transcribe( + self, + audio_bytes: bytes, + source: str, + file_name: str, + partial: bool, + ) -> str | None: + """提交一次音频快照并返回文本；返回 None 表示当前格式不支持中间转写。""" + prepared = prepare_audio_request(audio_bytes, source, file_name, partial) + if prepared is None: + return None + payload, upload_name, content_type = prepared + if self._session is None: + raise RuntimeError("model service is not started") + + form = FormData() + form.add_field("file", payload, filename=upload_name, content_type=content_type) + form.add_field("model", self.config.model) + form.add_field("response_format", "json") + headers = {"Authorization": f"Bearer {self.config.api_key}"} + endpoint = self.config.base_url.rstrip("/") + "/audio/transcriptions" + async with self._session.post(endpoint, data=form, headers=headers) as response: + body = await response.text() + if response.status >= 400: + raise RuntimeError(f"VLLM transcription failed ({response.status}): {body[:500]}") + try: + decoded: Any = await response.json(content_type=None) + except ValueError: + return body.strip() + if isinstance(decoded, dict): + return str(decoded.get("text") or decoded.get("transcript") or "").strip() + return str(decoded).strip() diff --git a/realtime_websocket/requirements.txt b/realtime_websocket/requirements.txt new file mode 100644 index 0000000..ed0d0a1 --- /dev/null +++ b/realtime_websocket/requirements.txt @@ -0,0 +1,2 @@ +aiohttp==3.11.11 +python-dotenv>=1.0 diff --git a/realtime_websocket/server.py b/realtime_websocket/server.py new file mode 100644 index 0000000..acc8e5f --- /dev/null +++ b/realtime_websocket/server.py @@ -0,0 +1,778 @@ +"""面向浏览器的独立 Qwen3-ASR VLLM WebSocket 编排服务。""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import logging +import os +import time +import webbrowser +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from uuid import uuid4 + +from aiohttp import WSMsgType, web +from dotenv import load_dotenv + +from auxiliary_service import AuxiliaryModelService, AuxiliaryServiceConfig +from model_service import ModelServiceConfig, VLLMTranscriptionService +from speaker_assembler import SegmentAssembler + + +# 与部署启动器读取同一配置；外部环境变量优先于 demo/.env。 +DEPLOY_ROOT = Path(__file__).resolve().parents[1] +load_dotenv(DEPLOY_ROOT / ".env") + +# 监听所有网卡，允许同一局域网内的浏览器访问服务器上的 Demo；端口集中在代码 +# 变量中维护，便于服务器部署时直接修改并保持页面和 WebSocket 使用一致端口。 +WEB_HOST = "0.0.0.0" +WEB_PORT = 8082 +WEB_DISPLAY_HOST = os.getenv("WEB_DISPLAY_HOST", "127.0.0.1") +DEFAULT_MODEL_SERVICE_URL = f"http://127.0.0.1:{os.getenv('VLLM_PORT', '9950')}/v1" +PARTIAL_BYTES_PER_SECOND = 16000 * 2 +VAD_FRAME_BYTES = 640 +VAD_FRAME_MS = 20 +VAD_SILENCE_MS = 800 +PARAGRAPH_SILENCE_MS = 1400 +VAD_RMS_THRESHOLD = 450 +MIN_SPEAKER_VOICE_MS = 800 +# 说话人分离开启时，用比普通 VAD 更短的静音作为候选话轮边界。 +# 这个边界只负责把 A→B 的短交接停顿拆开，不调用声纹模型，因此不会 +# 把窗口推理延迟叠加到音频帧处理路径；同一说话人的拆分片段仍由聚类合并。 +SPEAKER_GAP_MS = max(20, int(os.getenv("SPEAKER_GAP_MS", "400"))) +LOGGER = logging.getLogger(__name__) + + +class EndOfStream: + """带明确类型的队列结束标记，用于区分控制信号和真实音频字节。""" + + +EOF = EndOfStream() +MODEL_SERVICE_KEY = web.AppKey("model_service", VLLMTranscriptionService) +AUXILIARY_SERVICE_KEY = web.AppKey("auxiliary_service", AuxiliaryModelService) + + +@dataclass(frozen=True) +class SpeakerJob: + """等待辅助服务处理的单个已完成 turn；只保存该 turn 的 PCM 音频。""" + + sentence_id: int + audio: bytes + start_time_ms: float + end_time_ms: float + voiced_ms: float = 0.0 + + +def validate_model_service_url(value: str) -> str: + """只接受用户输入的 HTTP(S) VLLM 地址，并拒绝附带认证和查询参数的地址。""" + candidate = value.strip().rstrip("/") + parsed = urlparse(candidate) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("VLLM 地址必须是完整的 http:// 或 https:// URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("VLLM 地址不能包含账号、密码、查询参数或片段") + return candidate + + +@dataclass +class SessionMetrics: + """记录 WebSocket 会话耗时和结果修订次数，并在会话结束后展示。""" + + started_at: float + audio_bytes: int = 0 + input_chunks: int = 0 + partial_count: int = 0 + partial_revisions: int = 0 + first_partial_ms: float | None = None + final_ms: float | None = None + + def snapshot(self) -> dict[str, Any]: + """返回可安全序列化为 JSON 的指标，耗时均相对于会话开始时间计算。""" + now = time.perf_counter() + return { + "audio_bytes": self.audio_bytes, + "input_chunks": self.input_chunks, + "partial_count": self.partial_count, + "partial_revisions": self.partial_revisions, + "first_partial_ms": self.first_partial_ms, + "final_ms": self.final_ms, + "elapsed_ms": round((now - self.started_at) * 1000, 1), + } + + +class RealtimeSession: + """串行处理音频快照，并通过同一个 WebSocket 有序推送状态更新。""" + + def __init__( + self, + ws: web.WebSocketResponse, + model_service: VLLMTranscriptionService, + auxiliary_service: AuxiliaryModelService | None, + start: dict[str, Any], + ) -> None: + self.ws = ws + self.model_service = model_service + self.auxiliary_service = auxiliary_service + self.start = start + # 聚类状态只能属于当前连接，客户端复用 ID 不能串入另一会话的声纹池。 + self.session_id = uuid4().hex + self.send_lock = asyncio.Lock() + self.state_revision = 0 + self.audio_queue: asyncio.Queue[bytes | EndOfStream] = asyncio.Queue(maxsize=256) + self.speaker_queue: asyncio.Queue[SpeakerJob | EndOfStream] = asyncio.Queue(maxsize=64) + self.assembler = SegmentAssembler() + self.metrics = SessionMetrics(time.perf_counter()) + self.source = str(start.get("source") or "mic") + self.file_name = str(start.get("file_name") or "audio.wav") + self.windowed_partial = self.source == "mic" or Path(self.file_name).suffix.lower() in {".pcm", ".wav"} + self.sentence_strategy = int(start.get("sentence_strategy") or 0) + self.silence_limit_ms = PARAGRAPH_SILENCE_MS if self.sentence_strategy == 1 else VAD_SILENCE_MS + self.partial_interval_ms = max(300, int(start.get("partial_interval_ms") or 1200)) + self.max_segment_sec = max(2.0, float(start.get("max_segment_sec") or 12.0)) + self.merge_adjacent = self._parse_flag(start.get("display_merge"), True) + self.enable_native_partial = self._parse_flag(start.get("enable_native_partial_stream"), True) + self.segment_id = 0 + self.segment_audio = bytearray() + self.segment_start_ms = 0.0 + self.vad_buffer = bytearray() + self.processed_audio_bytes = 0 + self.silence_ms = 0 + self.in_speech = False + self.voiced_ms = 0.0 + self.pre_roll = bytearray() + self.wav_header_buffer = bytearray() + self.wav_payload_started = self.source != "file" or Path(self.file_name).suffix.lower() != ".wav" + self.wav_riff_read = False + self.wav_format_valid = False + self.wav_data_remaining: int | None = None + self.speaker_warning_sent = False + self.speaker_enabled = self._parse_flag(start.get("speaker_diarization"), True) + # 没有辅助服务时不提前切段，避免服务不可用时把一段语音拆成许多未知片段。 + self.speaker_gap_enabled = self.speaker_enabled and auxiliary_service is not None + raw_speaker_gap = start.get("speaker_gap_ms") + if raw_speaker_gap is None: + self.speaker_gap_ms = SPEAKER_GAP_MS + else: + try: + self.speaker_gap_ms = max(20, int(float(raw_speaker_gap))) + except (TypeError, ValueError, OverflowError): + self.speaker_gap_ms = SPEAKER_GAP_MS + self.input_stopped = False + + @staticmethod + def _parse_flag(value: Any, default: bool) -> bool: + """兼容前端传来的 0/1、布尔值和字符串开关，避免字符串 0 被误判为真。""" + if value is None: + return default + if isinstance(value, str): + return value.strip().lower() not in {"", "0", "false", "no", "off"} + return bool(value) + + async def emit(self, payload: dict[str, Any]) -> None: + """在连接仍然有效时发送一条有序事件，避免向已关闭连接写入数据。""" + async with self.send_lock: + if not self.ws.closed: + await self.ws.send_json(payload) + + async def emit_state(self, sentence: dict[str, Any] | None = None) -> None: + """每次状态更新后同时发送原始状态和重新计算的展示快照。""" + # 在首次 await 前冻结快照，音频 worker 与 speaker worker 不会混用两版状态。 + self.state_revision += 1 + state = { + "type": "display_state", "revision": self.state_revision, + "raw_segments": self.assembler.raw_snapshot(), + "display_blocks": self.assembler.display_blocks(self.merge_adjacent), + "metrics": self.metrics.snapshot(), + } + if sentence is not None: + await self.emit({"type": "sentences", "sentences": [sentence], "metrics": self.metrics.snapshot()}) + await self.emit(state) + + async def warn_speaker(self, message: str) -> None: + """只发送一次说话人服务告警，避免辅助服务异常时刷屏。""" + if self.speaker_warning_sent: + return + await self.emit( + { + "type": "speaker_warning", + "session_id": self.session_id, + "speaker_service_url": getattr(getattr(self.auxiliary_service, "config", None), "base_url", None), + "message": message, + } + ) + self.speaker_warning_sent = True + + def _duration_ms(self) -> float: + """根据 16 kHz PCM 字节数计算时长，不依赖前端可能漂移的时间戳。""" + return len(self.segment_audio) / PARTIAL_BYTES_PER_SECOND * 1000 + + async def _transcribe(self, partial: bool) -> str | None: + """通过 VLLM 适配器转写当前逻辑片段，并保留中间/最终请求的统一入口。""" + return await self.model_service.transcribe( + bytes(self.segment_audio), + "mic", + "turn.pcm", + partial=partial, + ) + + async def _emit_transcription(self, text: str, sentence_type: int, end_ms: float, commit_reason: str | None = None) -> None: + """写入或更新一条句子，确保中间结果和最终结果不会在前端产生重复行。""" + if not text: + return + sentence = self.assembler.apply_sentence( + { + "sentence_id": self.segment_id, + "sentence": text, + "sentence_type": sentence_type, + "start_time": self.segment_start_ms, + "end_time": end_ms, + "speaker_id": -1, + "speaker_name": "", + "speaker_evidence": "pending", + "speaker_confidence": 0.0, + "speaker_strategy": "vllm_no_speaker_evidence", + "commit_reason": commit_reason, + "speaker_status": ("queued" if sentence_type else "waiting_final") if self.speaker_enabled else "disabled", + "speaker_reason": ("等待声纹处理" if sentence_type else "语音片段结束后识别说话人") if self.speaker_enabled else "说话人分离已关闭", + } + ) + if sentence_type == 0: + self.metrics.partial_count += 1 + if sentence["revision_count"] > 0: + self.metrics.partial_revisions += 1 + if self.metrics.first_partial_ms is None: + self.metrics.first_partial_ms = round((time.perf_counter() - self.metrics.started_at) * 1000, 1) + else: + self.metrics.final_ms = round((time.perf_counter() - self.metrics.started_at) * 1000, 1) + await self.emit_state(sentence) + + @staticmethod + def _is_voice_frame(frame: bytes) -> bool: + """用 PCM 帧的 RMS 判断是否有语音，作为实时低延迟切句触发器。""" + if not frame: + return False + samples = memoryview(frame).cast("h") + if not samples: + return False + square_mean = sum(sample * sample for sample in samples) / len(samples) + return math.sqrt(square_mean) >= VAD_RMS_THRESHOLD + + def _strip_wav_header(self, chunk: bytes) -> bytes: + """增量解析 RIFF chunk；支持扩展头，并拒绝采样率或声道不匹配的 WAV。""" + if self.wav_payload_started: + if self.wav_data_remaining is None: + return chunk + payload = chunk[:self.wav_data_remaining] + self.wav_data_remaining -= len(payload) + return payload + self.wav_header_buffer.extend(chunk) + buffer = self.wav_header_buffer + if not self.wav_riff_read: + if len(buffer) < 12: + return b"" + if buffer[:4] != b"RIFF" or buffer[8:12] != b"WAVE": + raise ValueError("文件不是有效的 RIFF/WAV 音频") + del buffer[:12] + self.wav_riff_read = True + while len(buffer) >= 8: + kind = bytes(buffer[:4]) + size = int.from_bytes(buffer[4:8], "little") + if kind == b"data": + if not self.wav_format_valid or size % 2: + raise ValueError("WAV 必须为 16kHz、单声道、PCM16") + self.wav_data_remaining = size + self.wav_payload_started = True + payload = bytes(buffer[8:8 + size]) + self.wav_data_remaining -= len(payload) + buffer.clear() + return payload + if size > 1024 * 1024: + raise ValueError("WAV 元数据头过大，请转换为标准 PCM WAV") + chunk_size = 8 + size + (size % 2) + if len(buffer) < chunk_size: + return b"" + if kind == b"fmt ": + fmt = buffer[8:8 + size] + fields = (int.from_bytes(fmt[0:2], "little"), int.from_bytes(fmt[2:4], "little"), + int.from_bytes(fmt[4:8], "little"), int.from_bytes(fmt[14:16], "little")) + if size < 16 or fields != (1, 1, 16000, 16): + raise ValueError("WAV 必须为 16kHz、单声道、PCM16，请先转换音频") + self.wav_format_valid = True + del buffer[:chunk_size] + return b"" + + async def _resolve_speaker(self, job: SpeakerJob) -> None: + """异步解析单个 turn 的说话人，并把结果覆盖回同一个 sentence_id。""" + if not self.speaker_enabled: + return + async def update_status(status: str, reason: str) -> None: + """将每个失败或等待阶段回写原片段，避免只发一次全局告警。""" + updated = self.assembler.apply_speaker_update({ + "sentence_id": job.sentence_id, "speaker_id": -1, + "speaker_evidence": "pending", "speaker_confidence": 0.0, + "speaker_status": status, "speaker_reason": reason, + }) + await self.emit_state(updated) + + # 按有效有声帧检查长度，不能让句尾 800ms 静音把短插话伪装成长样本。 + if job.voiced_ms < MIN_SPEAKER_VOICE_MS: + await update_status("insufficient_audio", f"有效语音不足 {MIN_SPEAKER_VOICE_MS}ms，不继承上一位说话人") + return + if self.auxiliary_service is None: + await update_status("service_unavailable", "未配置说话人辅助模型服务") + await self.warn_speaker("未配置辅助模型服务，无法执行实时说话人分离") + return + await update_status("processing", "正在提取声纹并匹配说话人") + try: + speaker = await self.auxiliary_service.resolve_speaker( + job.audio, + self.session_id, + job.start_time_ms, + job.end_time_ms, + ) + except Exception as exc: + # 辅助服务异常不能阻断 ASR；当前片段继续保持 pending，方便定位服务问题。 + LOGGER.exception("speaker resolve failed: session=%s sentence=%s", self.session_id, job.sentence_id) + await update_status("service_error", str(exc)) + await self.warn_speaker(str(exc)) + return + if not speaker: + await update_status("no_embedding", "辅助服务未返回可用声纹结果") + return + update = dict(speaker) + update["sentence_id"] = job.sentence_id + update["speaker_name"] = str(update.get("speaker_name") or "") + updated = self.assembler.apply_speaker_update(update) + if updated is not None: + LOGGER.info("speaker result: session=%s sentence=%s status=%s strategy=%s", self.session_id, + job.sentence_id, updated.get("speaker_status"), updated.get("speaker_strategy")) + await self.emit_state(updated) + + async def process_speakers(self) -> None: + """按 turn 顺序串行访问辅助模型，保证在线聚类中心不会乱序更新。""" + while True: + item = await self.speaker_queue.get() + if isinstance(item, EndOfStream): + return + await self._resolve_speaker(item) + + async def _commit_segment(self, reason: str = "final") -> None: + """在 VAD 检测到一句结束后提交 final，并异步排队当前 turn 的说话人解析。""" + if not self.segment_audio or not self.in_speech: + return + # 去掉句尾触发切段的静音，ASR 与声纹都使用当前片段的真实有效范围。 + trailing_bytes = int(self.silence_ms * PARTIAL_BYTES_PER_SECOND / 1000) + if trailing_bytes: + del self.segment_audio[-trailing_bytes:] + final_audio = bytes(self.segment_audio) + final_start_ms = self.segment_start_ms + final_end_ms = self.segment_start_ms + self._duration_ms() + final_sentence_id = self.segment_id + text = await self._transcribe(partial=False) + if text: + await self._emit_transcription(text, 1, final_end_ms, reason) + if self.speaker_enabled: + await self.speaker_queue.put( + SpeakerJob( + sentence_id=final_sentence_id, + audio=final_audio, + start_time_ms=final_start_ms, + end_time_ms=final_end_ms, + voiced_ms=self.voiced_ms, + ) + ) + elif self.assembler.segments.pop(final_sentence_id, None) is not None: + # final 判定无文本时撤回临时结果，不能留下永远等待声纹的 partial。 + await self.emit_state() + self.segment_audio.clear() + self.segment_start_ms = self.processed_audio_bytes / PARTIAL_BYTES_PER_SECOND * 1000 + self.segment_id += 1 + self.silence_ms = 0 + self.in_speech = False + self.voiced_ms = 0.0 + + async def process_audio(self) -> None: + """消费音频，以 VAD 静音结束作为切句主逻辑，并按窗口发送 partial。""" + last_partial_bytes = 0 + partial_bytes = int(self.partial_interval_ms / 1000 * PARTIAL_BYTES_PER_SECOND) + while True: + item = await self.audio_queue.get() + if isinstance(item, EndOfStream): + break + chunk = self._strip_wav_header(item) if self.source == "file" else item + self.metrics.input_chunks += 1 + if not self.windowed_partial: + # websocket_handler 已拒绝压缩文件；这里保留防御分支，避免未来 + # 新客户端绕过入口时又悄悄退化成“整段上传后切片”。 + raise RuntimeError("实时流式模式只接受 16kHz PCM16 音频") + if not chunk: + continue + self.vad_buffer.extend(chunk) + while len(self.vad_buffer) >= VAD_FRAME_BYTES: + frame = bytes(self.vad_buffer[:VAD_FRAME_BYTES]) + del self.vad_buffer[:VAD_FRAME_BYTES] + frame_start_ms = self.processed_audio_bytes / PARTIAL_BYTES_PER_SECOND * 1000 + self.processed_audio_bytes += len(frame) + self.metrics.audio_bytes += len(frame) + voiced = self._is_voice_frame(frame) + if voiced and not self.in_speech: + self.in_speech = True + self.segment_start_ms = frame_start_ms - len(self.pre_roll) / PARTIAL_BYTES_PER_SECOND * 1000 + self.segment_audio = bytearray(self.pre_roll) + self.pre_roll.clear() + last_partial_bytes = 0 + if self.in_speech: + self.segment_audio.extend(frame) + if voiced: + self.voiced_ms += VAD_FRAME_MS + self.silence_ms = 0 if voiced else self.silence_ms + VAD_FRAME_MS + if len(self.segment_audio) - last_partial_bytes >= partial_bytes and self.silence_ms < self.silence_limit_ms: + text = await self._transcribe(partial=True) + if text: + await self._emit_transcription(text, 0, self.segment_start_ms + self._duration_ms()) + last_partial_bytes = len(self.segment_audio) + # 短交接停顿优先于普通 800/1400ms 静音切段，但必须先有 + # 至少 800ms 有效语音，避免把咳嗽、噪声或极短插话送去聚类。 + short_speaker_gap = ( + self.speaker_gap_enabled + and self.voiced_ms >= MIN_SPEAKER_VOICE_MS + and self.silence_ms >= self.speaker_gap_ms + ) + if short_speaker_gap or self.silence_ms >= self.silence_limit_ms or self._duration_ms() >= self.max_segment_sec * 1000: + reason = ( + "speaker_gap" if short_speaker_gap + else "silence" if self.silence_ms >= self.silence_limit_ms + else "max_duration" + ) + await self._commit_segment(reason) + last_partial_bytes = 0 + else: + # 参考原 WebSocket 保留 200ms 前滚，减少首字低能量音素被裁掉。 + self.pre_roll.extend(frame) + del self.pre_roll[:-6400] + + if not self.wav_payload_started or (not self.input_stopped and self.wav_data_remaining not in (None, 0)): + raise ValueError("WAV 文件不完整，未收到全部音频数据") + if len(self.vad_buffer) % 2: + raise ValueError("PCM16 音频必须包含完整的双字节采样") + if self.windowed_partial and self.vad_buffer: + tail_ms = len(self.vad_buffer) / PARTIAL_BYTES_PER_SECOND * 1000 + tail_voiced = self._is_voice_frame(bytes(self.vad_buffer)) + if tail_voiced and not self.in_speech: + self.in_speech = True + self.segment_start_ms = self.processed_audio_bytes / PARTIAL_BYTES_PER_SECOND * 1000 + self.processed_audio_bytes += len(self.vad_buffer) + self.metrics.audio_bytes += len(self.vad_buffer) + if self.in_speech: + self.segment_audio.extend(self.vad_buffer) + self.silence_ms = 0 if tail_voiced else self.silence_ms + tail_ms + self.voiced_ms += tail_ms if tail_voiced else 0 + self.vad_buffer.clear() + if self.segment_audio: + if not self.windowed_partial: + self.in_speech = True + await self._commit_segment() + await self.emit({"type": "metrics", "metrics": self.metrics.snapshot()}) + + +def deployment_model_name() -> str: + """把部署脚本的 0.6b/1.7b 别名解析成 vLLM 对外发布的模型名。""" + public_name = os.getenv("VLLM_SERVED_MODEL_NAME") + if public_name: + return public_name + requested = os.getenv("QWEN3_ASR_MODEL", "default") + with (DEPLOY_ROOT / "model_manifest.json").open(encoding="utf-8") as source: + manifest = json.load(source) + if requested == "default": + return str(manifest["default_model"]) + for model_id, config in manifest["models"].items(): + if requested.lower() == str(config.get("alias", "")).lower(): + return model_id + return requested + + +def parse_args() -> argparse.Namespace: + """只解析服务选择参数；浏览器服务端口继续由代码内部变量统一维护。""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-service-url", default=os.getenv("MODEL_SERVICE_URL", DEFAULT_MODEL_SERVICE_URL)) + parser.add_argument("--model", default=deployment_model_name()) + parser.add_argument("--no-browser", action="store_true") + return parser.parse_args() + + +async def index_handler(_: web.Request) -> web.FileResponse: + """返回独立 Demo 测试页面，并避免入口页缓存旧的脚本版本号。""" + # 入口页必须每次重新校验，配合 app.js 的版本号变更，避免用户继续运行旧前端。 + return web.FileResponse( + Path(__file__).parent / "static" / "index.html", + headers={"Cache-Control": "no-store"}, + ) + + +async def config_handler(request: web.Request) -> web.Response: + """暴露服务启动时的默认配置，让页面自动填充 VLLM 地址和模型名。""" + config = request.app[MODEL_SERVICE_KEY].config + auxiliary = request.app.get(AUXILIARY_SERVICE_KEY) + return web.json_response({ + "model_service_url": config.base_url, "model": config.model, + "speaker_service_url": getattr(getattr(auxiliary, "config", None), "base_url", None), + }) + + +async def websocket_handler(request: web.Request) -> web.WebSocketResponse: + """处理一个浏览器会话，每个连接独立保存音频、句子和展示状态。""" + ws = web.WebSocketResponse(max_msg_size=64 * 1024 * 1024) + await ws.prepare(request) + default_model_service: VLLMTranscriptionService = request.app[MODEL_SERVICE_KEY] + model_service = default_model_service + owns_model_service = False + processing: asyncio.Task[None] | None = None + speaker_processing: asyncio.Task[None] | None = None + session: RealtimeSession | None = None + try: + first = await ws.receive() + if first.type != WSMsgType.TEXT: + await ws.send_json({"type": "error", "message": "first message must be JSON start"}) + return ws + try: + start = json.loads(first.data) + except json.JSONDecodeError: + await ws.send_json({"type": "error", "message": "invalid start JSON"}) + return ws + if not isinstance(start, dict) or start.get("type") != "start": + await ws.send_json({"type": "error", "message": "first message must have type=start"}) + return ws + + # 本项目用于验证实时流式链路，文件模式只接受可以按 PCM 帧连续处理的 + # WAV/PCM；MP3、M4A 等压缩容器只能在文件完整到达后解码，不纳入本次测试。 + source = str(start.get("source") or "mic") + file_suffix = Path(str(start.get("file_name") or "")).suffix.lower() + if source == "file" and file_suffix not in {".pcm", ".wav"}: + await ws.send_json( + { + "type": "error", + "message": "实时流式测试的文件模式只支持 PCM 或 WAV，请改用麦克风、PCM 或 WAV", + } + ) + return ws + + # 页面可以在不重启 WebSocket Demo 的情况下为当前会话切换 VLLM 地址； + # 未切换时继续复用默认服务，避免普通场景为每个连接重复创建 HTTP 会话。 + try: + requested_url = validate_model_service_url( + str(start.get("model_service_url") or default_model_service.config.base_url) + ) + except ValueError as exc: + await ws.send_json({"type": "error", "message": str(exc)}) + return ws + requested_model = str(start.get("model") or default_model_service.config.model).strip() + if ( + requested_url != default_model_service.config.base_url + or requested_model != default_model_service.config.model + ): + model_service = VLLMTranscriptionService( + ModelServiceConfig(base_url=requested_url, model=requested_model) + ) + await model_service.start() + owns_model_service = True + + auxiliary_service = request.app.get(AUXILIARY_SERVICE_KEY) + session = RealtimeSession(ws, model_service, auxiliary_service, start) + auxiliary_config = getattr(auxiliary_service, "config", None) + speaker_health: dict[str, Any] | None = None + speaker_health_error: str | None = None + if session.speaker_enabled and auxiliary_service is not None: + # 健康检查只用于尽早暴露辅助服务问题；即使失败也不阻断 ASR， + # 这样可以从同一页面继续观察 ASR 与说话人链路的差异。 + health_check = getattr(auxiliary_service, "health", None) + if callable(health_check): + try: + speaker_health = await asyncio.wait_for(health_check(), timeout=5) + if speaker_health.get("speaker_embedding_ready", speaker_health.get("ready")) is False: + speaker_health_error = "辅助模型服务未就绪，请检查 /health 返回的 models 状态" + except Exception as exc: + speaker_health_error = f"说话人辅助服务不可用：{exc}" + if speaker_health_error: + # 辅助服务未就绪时先保留原始 VAD 切段，避免在没有声纹结果的 + # 情况下增加大量短片段；服务恢复后由新的会话重新启用。 + session.speaker_gap_enabled = False + await session.emit( + { + "type": "start", + "model_service_url": model_service.config.base_url, + "model": model_service.config.model, + "session_id": session.session_id, + "enable_native_partial_stream": session.enable_native_partial, + "native_partial_supported": model_service.native_partial_supported, + "partial_mode": "http_cumulative_window", + "speaker_diarization_enabled": session.speaker_enabled, + "speaker_service_url": getattr(auxiliary_config, "base_url", None), + "speaker_service_health": speaker_health, + "speaker_gap_enabled": session.speaker_gap_enabled, + "speaker_gap_ms": session.speaker_gap_ms, + "sentence_strategy": session.sentence_strategy, + "silence_limit_ms": session.silence_limit_ms, + "display_state_supported": True, + } + ) + if speaker_health_error: + await session.warn_speaker(speaker_health_error) + processing = asyncio.create_task(session.process_audio()) + if session.speaker_enabled: + # speaker worker 与音频处理并行运行；它只消费已经结束的 turn， + # 因此不会阻塞下一帧音频进入队列或影响 ASR partial 输出。 + speaker_processing = asyncio.create_task(session.process_speakers()) + async def guarded(operation): + """接收和队列背压同时监听 worker，推理失败立即报错而非永远等 stop。""" + pending = asyncio.create_task(operation) + try: + workers = [task for task in (processing, speaker_processing) if task is not None] + done, _ = await asyncio.wait([pending, *workers], return_when=asyncio.FIRST_COMPLETED) + if pending in done: + return await pending + for worker in workers: + if worker in done: + await worker + raise RuntimeError("实时处理任务意外结束") + return await pending + finally: + if not pending.done(): + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + + input_finished = False + while not ws.closed: + message = await guarded(ws.receive()) + if message.type == WSMsgType.BINARY: + await guarded(session.audio_queue.put(bytes(message.data))) + continue + if message.type == WSMsgType.TEXT: + try: + control = json.loads(message.data) + except json.JSONDecodeError: + continue + if not isinstance(control, dict): + continue + if control.get("type") in {"eof", "stop"}: + input_finished = True + session.input_stopped = control.get("type") == "stop" + await session.emit({"type": "draining", "message": "正在完成转写和说话人识别"}) + await guarded(session.audio_queue.put(EOF)) + break + if control.get("type") == "abort": + processing.cancel() + if speaker_processing is not None: + speaker_processing.cancel() + await asyncio.gather( + processing, + *(task for task in [speaker_processing] if task is not None), + return_exceptions=True, + ) + return ws + if message.type in {WSMsgType.ERROR, WSMsgType.CLOSE, WSMsgType.CLOSED}: + processing.cancel() + if speaker_processing is not None: + speaker_processing.cancel() + await asyncio.gather( + processing, + *(task for task in [speaker_processing] if task is not None), + return_exceptions=True, + ) + return ws + if not input_finished: + # 浏览器或网络断开后，音频生产者已经不存在，不能让处理任务继续等待 + # 永远不会到来的 EOF，因此这里主动取消任务并回收异常结果。 + processing.cancel() + if speaker_processing is not None: + speaker_processing.cancel() + await asyncio.gather( + processing, + *(task for task in [speaker_processing] if task is not None), + return_exceptions=True, + ) + return ws + try: + await processing + if speaker_processing is not None: + await session.speaker_queue.put(EOF) + await speaker_processing + await session.emit_state() + await session.emit({ + "type": "end", "metrics": session.metrics.snapshot(), + "sentences": session.assembler.raw_snapshot(), + "display_blocks": session.assembler.display_blocks(session.merge_adjacent), + }) + except asyncio.CancelledError: + raise + except Exception as exc: + await session.emit({"type": "error", "message": str(exc)}) + except Exception as exc: + LOGGER.exception("WebSocket session failed") + if not ws.closed: + await ws.send_json({"type": "error", "message": str(exc)}) + finally: + for task in (processing, speaker_processing): + if task is not None and not task.done(): + task.cancel() + pending_tasks = [task for task in (processing, speaker_processing) if task is not None] + if pending_tasks: + await asyncio.gather(*pending_tasks, return_exceptions=True) + # stop、abort、断线和推理异常均释放会话，清理失败不覆盖最终识别结果。 + if session is not None: + reset = getattr(session.auxiliary_service, "reset_speaker_session", None) + if reset is not None: + try: + await asyncio.wait_for(reset(session.session_id), timeout=5) + except Exception: + LOGGER.warning("speaker session cleanup failed: %s", session.session_id, exc_info=True) + if owns_model_service: + await model_service.close() + if not ws.closed: + await ws.close() + return ws + + +async def start_app(model_service_url: str, model: str) -> web.Application: + """创建 HTTP/WebSocket 应用，并挂载可复用的 VLLM 适配器。""" + app = web.Application() + app[MODEL_SERVICE_KEY] = VLLMTranscriptionService(ModelServiceConfig(base_url=model_service_url, model=model)) + app[AUXILIARY_SERVICE_KEY] = AuxiliaryModelService( + AuxiliaryServiceConfig(base_url=os.getenv("AUXILIARY_SERVICE_URL", "http://127.0.0.1:8010")) + ) + + async def lifecycle(application: web.Application): + await application[MODEL_SERVICE_KEY].start() + await application[AUXILIARY_SERVICE_KEY].start() + yield + await application[AUXILIARY_SERVICE_KEY].close() + await application[MODEL_SERVICE_KEY].close() + + app.cleanup_ctx.append(lifecycle) + app.router.add_get("/", index_handler) + app.router.add_get("/api/config", config_handler) + app.router.add_static("/static/", Path(__file__).parent / "static") + app.router.add_get("/ws", websocket_handler) + # 参考腾讯 Demo 将 static 目录挂载到根路径；页面中的 style.css 和 app.js + # 使用相对地址，必须同时提供根路径静态资源路由，否则浏览器会显示无样式页面。 + app.router.add_static("/", Path(__file__).parent / "static", show_index=False) + return app + + +def main() -> None: + """启动本地测试页面和 WebSocket 服务。""" + args = parse_args() + logging.basicConfig(level=logging.INFO) + if not args.no_browser: + webbrowser.open(f"http://{WEB_DISPLAY_HOST}:{WEB_PORT}/") + print(f"WebSocket demo: http://{WEB_DISPLAY_HOST}:{WEB_PORT}/", flush=True) + print(f"VLLM service: {args.model_service_url} ({args.model})", flush=True) + web.run_app(start_app(args.model_service_url, args.model), host=WEB_HOST, port=WEB_PORT) + + +if __name__ == "__main__": + main() diff --git a/realtime_websocket/speaker_assembler.py b/realtime_websocket/speaker_assembler.py new file mode 100644 index 0000000..a9ae9c3 --- /dev/null +++ b/realtime_websocket/speaker_assembler.py @@ -0,0 +1,186 @@ +"""独立 Demo 使用的有状态原始片段与展示区块组装器。""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass, field +import math +from typing import Any + + +TRUSTED_CONFIDENCE = 0.6 + + +def _as_float(value: Any, default: float = 0.0) -> float: + try: + parsed = float(value) + return parsed if math.isfinite(parsed) else default + except (TypeError, ValueError, OverflowError): + return default + + +def _as_int(value: Any, default: int = -1) -> int: + try: + return int(value) + except (TypeError, ValueError, OverflowError): + return default + + +def _without_embeddings(payload: dict[str, Any]) -> dict[str, Any]: + """阻止原始声纹向量进入 Demo 的持久化状态。""" + sanitized: dict[str, Any] = {} + for key, value in payload.items(): + lowered = str(key).lower() + if "embedding" in lowered or lowered in {"_chunks", "chunks", "_chunk_embeddings"}: + continue + sanitized[key] = deepcopy(value) + return sanitized + + +def _is_trusted(segment: dict[str, Any]) -> bool: + """在展示身份稳定前，要求存在独立的说话人证据。""" + evidence = str(segment.get("speaker_evidence") or "pending").lower() + return ( + _as_int(segment.get("speaker_id")) >= 0 + and evidence in {"fresh", "confirmed"} + and _as_float(segment.get("speaker_confidence")) >= TRUSTED_CONFIDENCE + and segment.get("speaker_strategy") not in {"short_attach", "embedding_attach"} + ) + + +def _speaker_identity(segment: dict[str, Any]) -> tuple[Any, ...]: + """实名身份优先于匿名簇，防止同簇弱标签或不同实名被合并。""" + for key in ("user_id", "registry_speaker_id"): + if segment.get(key) not in (None, ""): + return (key, str(segment[key])) + return ("cluster", segment.get("speaker_id"), segment.get("speaker_name", "")) + + +@dataclass +class SegmentAssembler: + """保存可幂等更新的原始片段，并按时间顺序派生展示区块。""" + + segments: dict[int, dict[str, Any]] = field(default_factory=dict) + + def apply_sentence(self, incoming: dict[str, Any]) -> dict[str, Any]: + """写入或更新一条中间或最终句子，并保护不可靠的短身份名称。""" + sentence_id = _as_int(incoming.get("sentence_id"), 0) + previous = self.segments.get(sentence_id) + segment = dict(previous or {}) + segment.update(_without_embeddings(incoming)) + # 文本重发不能抹掉已到达的声纹更新，也不能把 final 回滚成 partial。 + if previous and previous.get("sentence_type") == 1 and incoming.get("sentence_type") == 0: + return deepcopy(previous) + segment["sentence_id"] = sentence_id + segment["sentence"] = str(segment.get("sentence") or segment.get("text") or "").strip() + segment["sentence_type"] = _as_int(segment.get("sentence_type"), 0) + segment["start_time"] = _as_float(segment.get("start_time")) + segment["end_time"] = _as_float(segment.get("end_time")) + segment["speaker_id"] = _as_int(segment.get("speaker_id")) + segment["speaker_name"] = str(segment.get("speaker_name") or "") + segment["speaker_evidence"] = str(segment.get("speaker_evidence") or "pending") + segment["speaker_confidence"] = _as_float(segment.get("speaker_confidence")) + + strategy = str(segment.get("speaker_strategy") or "") + if strategy in {"short_attach", "embedding_attach"}: + # 继承而来的短名称不属于新的可靠证据，必须继续保持 pending 状态， + # 防止前一个片段的身份错误污染当前展示结果。 + segment["speaker_id"] = -1 + segment["speaker_name"] = "" + segment["speaker_evidence"] = "pending" + segment["speaker_confidence"] = 0.0 + segment["speaker_status"] = "inherited_rejected" + segment["speaker_reason"] = "缺少当前片段的独立声纹证据" + segment.pop("user_id", None) + segment.pop("registry_speaker_id", None) + + if previous is not None: + segment["revision_count"] = int(previous.get("revision_count", 0)) + ( + 1 if segment["sentence"] != previous.get("sentence") else 0 + ) + else: + segment["revision_count"] = 0 + self.segments[sentence_id] = segment + return deepcopy(segment) + + def apply_speaker_update(self, update: dict[str, Any]) -> dict[str, Any] | None: + """只将明确标记为新鲜或已确认的更新应用到已有片段。""" + sentence_id = _as_int(update.get("sentence_id"), -1) + current = self.segments.get(sentence_id) + if current is None: + return None + candidate = dict(current) + # 新身份的实名字段必须来自本次证据，不能沿用同片段旧识别的人员 ID。 + candidate.pop("user_id", None) + candidate.pop("registry_speaker_id", None) + # 说话人响应只能更新身份字段，不能篡改已确认的文本和 ASR 时间范围。 + candidate.update({ + key: value for key, value in _without_embeddings(update).items() + if key.startswith("speaker_") or key in {"user_id", "registry_speaker_id"} + }) + candidate["speaker_id"] = _as_int(candidate.get("speaker_id")) + candidate["speaker_evidence"] = str(update.get("speaker_evidence") or "pending") + candidate["speaker_confidence"] = _as_float(update.get("speaker_confidence")) + if not _is_trusted(candidate): + # 保留拒绝原因供前端诊断，但绝不把不可靠身份带入展示或历史状态。 + candidate.update(speaker_id=-1, speaker_name="", speaker_evidence="pending", speaker_confidence=0.0) + candidate.pop("user_id", None) + candidate.pop("registry_speaker_id", None) + if update.get("speaker_status") not in { + "queued", "processing", "waiting_final", "disabled", "insufficient_audio", + "service_unavailable", "service_error", "no_embedding", "evidence_rejected", + }: + candidate["speaker_status"] = "evidence_rejected" + candidate["speaker_reason"] = "声纹结果缺少新鲜证据或置信度不足" + else: + candidate["speaker_status"] = "confirmed" + candidate["speaker_reason"] = "当前片段声纹已确认" + self.segments[sentence_id] = candidate + return deepcopy(candidate) + + def raw_snapshot(self) -> list[dict[str, Any]]: + """返回按时间、再按句子 ID 排序后的全部原始片段。""" + return [ + deepcopy(segment) + for segment in sorted(self.segments.values(), key=lambda item: (item["start_time"], item["sentence_id"])) + ] + + def display_blocks(self, merge_adjacent: bool = True) -> list[dict[str, Any]]: + """生成展示区块，同时保持非相邻说话人轮次的原始顺序。""" + blocks: list[dict[str, Any]] = [] + for segment in self.raw_snapshot(): + trusted = _is_trusted(segment) + identity_key = _speaker_identity(segment) if trusted else ("pending", segment["sentence_id"]) + if ( + merge_adjacent + and blocks + and trusted + and blocks[-1].get("identity_key") == identity_key + ): + block = blocks[-1] + block["sentence"] = f'{block["sentence"]} {segment["sentence"]}'.strip() + block["end_time"] = max(block["end_time"], segment["end_time"]) + block["segment_ids"].append(segment["sentence_id"]) + block["sentence_type"] = min(block["sentence_type"], segment["sentence_type"]) + continue + + blocks.append( + { + "block_id": f"block-{segment['sentence_id']}", + "sentence": segment["sentence"], + "start_time": segment["start_time"], + "end_time": segment["end_time"], + "segment_ids": [segment["sentence_id"]], + "speaker_id": segment["speaker_id"] if trusted else -1, + "speaker_name": segment["speaker_name"] if trusted else "", + "speaker_evidence": "confirmed" if trusted else "pending", + "speaker_status": segment.get("speaker_status", "pending"), + "speaker_reason": segment.get("speaker_reason", ""), + "speaker_confidence": segment.get("speaker_confidence", 0.0), + "sentence_type": segment["sentence_type"], + "identity_key": identity_key, + } + ) + for block in blocks: + block.pop("identity_key", None) + return blocks diff --git a/realtime_websocket/static/app.js b/realtime_websocket/static/app.js new file mode 100644 index 0000000..944df74 --- /dev/null +++ b/realtime_websocket/static/app.js @@ -0,0 +1,743 @@ +// ===== 页面元素 ===== +const elEngineModel = document.getElementById('engineModel'); +const elModelServiceUrl = document.getElementById('modelServiceUrl'); +const elSpeakerStatus = document.getElementById('speakerStatus'); +const elDisplayMerge = document.getElementById('displayMerge'); +const elSpeakerDiarization = document.getElementById('speakerDiarization'); +const elDiarizationLabel = document.getElementById('diarizationLabel'); +const elSentenceStrategy = document.getElementById('sentenceStrategy'); +const elBtnStart = document.getElementById('btnStart'); +const elBtnStop = document.getElementById('btnStop'); +const elResultMeta = document.getElementById('resultMeta'); +const elVoiceIdDisplay = document.getElementById('voiceIdDisplay'); +const elBtnCopyVoiceId = document.getElementById('btnCopyVoiceId'); +const elBtnExportWav = document.getElementById('btnExportWav'); +const elStatusDot = document.getElementById('statusDot'); +const elStatusText = document.getElementById('statusText'); +const elResultArea = document.getElementById('resultArea'); +const elResultPlaceholder = document.getElementById('resultPlaceholder'); +const elLogArea = document.getElementById('logArea'); +const elBtnClearLog = document.getElementById('btnClearLog'); +const elMicStatus = document.getElementById('micStatus'); +const elMicTimer = document.getElementById('micTimer'); +const elMicElapsed = document.getElementById('micElapsed'); + +// 输入模式标签页 +const elTabMic = document.getElementById('tabMic'); +const elTabFile = document.getElementById('tabFile'); +const elPanelMic = document.getElementById('panelMic'); +const elPanelFile = document.getElementById('panelFile'); + +// 文件选择区域 +const elAudioFile = document.getElementById('audioFile'); +const elFileInfo = document.getElementById('fileInfo'); +const elAudioMeta = document.getElementById('audioMeta'); +const elMetaFormat = document.getElementById('metaFormat'); +const elMetaSampleRate = document.getElementById('metaSampleRate'); +const elMetaDuration = document.getElementById('metaDuration'); +const elSpeedControl = document.getElementById('speedControl'); +const elSpeedSlider = document.getElementById('speedSlider'); +const elSpeedValue = document.getElementById('speedValue'); + +// ===== 说话人分离开关 ===== +elSpeakerDiarization.addEventListener('change', () => { + elDiarizationLabel.textContent = elSpeakerDiarization.checked ? '开启' : '关闭'; +}); + +// ===== 日志区域 ===== +elBtnClearLog.addEventListener('click', () => { elLogArea.innerHTML = ''; }); + +function appendLog(msg) { + const now = new Date(); + const ts = [now.getHours(), now.getMinutes(), now.getSeconds()] + .map(n => String(n).padStart(2, '0')).join(':') + + '.' + String(now.getMilliseconds()).padStart(3, '0'); + const typeClass = 'log-type-' + (msg.type || 'unknown'); + const entry = document.createElement('div'); + entry.className = 'log-entry'; + // 原始文本不作为 HTML 解释，转写中的标签也应原样显示。 + const stamp = document.createElement('span'); + stamp.className = 'log-time'; + stamp.textContent = ts; + const content = document.createElement('span'); + content.className = typeClass; + content.textContent = JSON.stringify(msg); + entry.append(stamp, content); + elLogArea.appendChild(entry); + while (elLogArea.childNodes.length > 300) elLogArea.firstChild.remove(); + elLogArea.scrollTop = elLogArea.scrollHeight; +} + +// ===== 会话状态 ===== +let ws = null; +let sending = false; +let stoppingByUser = false; +let currentVoiceId = null; +let audioChunks = []; +let sessionId = 0; + +let micStream = null; +let micAudioContext = null; +let micWorklet = null; +let micTimerInterval = null; +let micStartTime = 0; + +// 输入模式（麦克风 / 文件） +let inputMode = 'mic'; +let selectedFile = null; + +// 文件扩展名 → SDK VoiceFormat 常量 +const EXT_FORMAT_MAP = { + 'pcm': 1, 'wav': 12, 'mp3': 8, 'm4a': 14, + 'aac': 16, 'opus': 10, 'ogg': 10, 'silk': 6, 'speex': 4 +}; +// PCM/WAV 的默认发送倍速；实时验证默认按 1 倍速输入。 +const DEFAULT_SPEED = { + 'pcm': 1.0, 'wav': 1.0, + 'mp3': 2.0, 'm4a': 2.0, 'aac': 2.0, + 'opus': 3.0, 'ogg': 3.0, 'silk': 3.0, 'speex': 3.0 +}; +const MAX_SPEED = 3.0; +// 实时 WebSocket 需要服务端逐帧读取音频；压缩格式必须等文件完整后才能解码， +// 因此本次流式验证只允许 PCM/WAV，避免把整段上传伪装成实时识别。 +const STREAMABLE_AUDIO_EXTENSIONS = new Set(['pcm', 'wav']); + +const SPEAKER_COLORS = ['#4a7dff', '#52c41a', '#faad14', '#ff4d4f', '#9254de', '#13c2c2']; + +let sentenceMap = {}; +let speakerOrderMap = {}; +let speakerOrderCounter = 0; +let displayStateSupported = false; +let displayRevision = -1; + +// ===== 输入模式标签页 ===== +function switchMode(mode) { + if (ws) return; + inputMode = mode; + elTabMic.classList.toggle('active', mode === 'mic'); + elTabFile.classList.toggle('active', mode === 'file'); + elPanelMic.style.display = mode === 'mic' ? '' : 'none'; + elPanelFile.style.display = mode === 'file' ? '' : 'none'; + if (mode === 'file') { + elBtnStart.disabled = !selectedFile; + } else { + elBtnStart.disabled = false; + } +} +elTabMic.addEventListener('click', () => switchMode('mic')); +elTabFile.addEventListener('click', () => switchMode('file')); + +// ===== 文件选择 ===== +elAudioFile.addEventListener('change', (e) => { + const file = e.target.files[0]; + if (!file) return; + const ext = getFileExt(file.name); + if (!STREAMABLE_AUDIO_EXTENSIONS.has(ext)) { + selectedFile = null; + e.target.value = ''; + elFileInfo.textContent = '实时测试只支持 PCM 或 WAV，请先转换音频格式'; + elFileInfo.classList.remove('has-file'); + elAudioMeta.style.display = 'none'; + elSpeedControl.style.display = 'none'; + elBtnStart.disabled = true; + showToast('压缩音频不能按当前实时 WebSocket 逐帧识别，请转成 PCM 或 WAV', true); + return; + } + selectedFile = file; + const sizeMB = (file.size / 1024 / 1024).toFixed(1); + elFileInfo.textContent = `${file.name} (${sizeMB}MB)`; + elFileInfo.classList.add('has-file'); + elBtnStart.disabled = false; + parseAudioMeta(file); +}); + +// 发送速度滑块 +elSpeedSlider.addEventListener('input', () => { + elSpeedValue.textContent = parseFloat(elSpeedSlider.value).toFixed(1) + 'x'; +}); + +// ===== 音频元数据解析 ===== +function getFileExt(filename) { + const parts = filename.split('.'); + return parts.length > 1 ? parts[parts.length - 1].toLowerCase() : ''; +} + +function formatDuration(seconds) { + if (!seconds || !isFinite(seconds)) return '—'; + const min = Math.floor(seconds / 60); + const sec = Math.floor(seconds % 60); + const ms = Math.round((seconds % 1) * 10); + return `${String(min).padStart(2, '0')}:${String(sec).padStart(2, '0')}.${ms}`; +} + +async function parseAudioMeta(file) { + const ext = getFileExt(file.name); + elMetaFormat.textContent = ext.toUpperCase(); + // 根据格式设默认倍速 + const defaultSpeed = Math.min(DEFAULT_SPEED[ext] || 1.0, MAX_SPEED); + elSpeedSlider.value = defaultSpeed; + elSpeedValue.textContent = defaultSpeed.toFixed(1) + 'x'; + elAudioMeta.style.display = ''; + elSpeedControl.style.display = ''; + + if (ext === 'pcm') { + elMetaSampleRate.textContent = '16000 Hz (假设)'; + const durationSec = file.size / (16000 * 2 * 1); + elMetaDuration.textContent = formatDuration(durationSec) + ' (估算)'; + } else if (ext === 'wav') { + try { + const headerBuf = await file.slice(0, 44).arrayBuffer(); + const view = new DataView(headerBuf); + const sampleRate = view.getUint32(24, true); + const bitsPerSample = view.getUint16(34, true); + const numChannels = view.getUint16(22, true); + const byteRate = sampleRate * numChannels * (bitsPerSample / 8); + const dataSize = file.size - 44; + const durationSec = dataSize / byteRate; + elMetaSampleRate.textContent = sampleRate + ' Hz, ' + bitsPerSample + 'bit, ' + numChannels + 'ch'; + elMetaDuration.textContent = formatDuration(durationSec); + } catch { + elMetaSampleRate.textContent = '解析失败'; + elMetaDuration.textContent = '—'; + } + } else { + elMetaSampleRate.textContent = '—'; + elMetaDuration.textContent = '解析中...'; + try { + const url = URL.createObjectURL(file); + const audio = new Audio(); + audio.preload = 'metadata'; + audio.src = url; + await new Promise((resolve, reject) => { + audio.onloadedmetadata = resolve; + audio.onerror = reject; + setTimeout(reject, 5000); + }); + elMetaDuration.textContent = formatDuration(audio.duration); + URL.revokeObjectURL(url); + } catch { + elMetaDuration.textContent = '无法解析'; + } + } +} + +// ===== 复制和提示 ===== +function showToast(message, isError) { + const toast = document.createElement('div'); + toast.className = 'copy-toast' + (isError ? ' copy-toast-error' : ''); + toast.textContent = message; + document.body.appendChild(toast); + requestAnimationFrame(() => { toast.classList.add('visible'); }); + setTimeout(() => { + toast.classList.remove('visible'); + setTimeout(() => toast.remove(), 300); + }, 1500); +} + +function handleCopyClick(btn, textEl) { + const text = textEl.textContent.trim(); + if (!text || text === '—') { showToast('暂无内容可复制', true); return; } + navigator.clipboard.writeText(text).then(() => { + btn.textContent = '✅'; + showToast('已复制到剪贴板'); + setTimeout(() => { btn.textContent = '📋'; }, 1500); + }).catch(() => showToast('复制失败', true)); +} + +elBtnCopyVoiceId.addEventListener('click', () => handleCopyClick(elBtnCopyVoiceId, elVoiceIdDisplay)); + +// ===== WAV 导出 ===== +function buildWavBlob(pcmChunks) { + let totalLen = 0; + for (const c of pcmChunks) totalLen += c.byteLength; + const buf = new ArrayBuffer(44 + totalLen); + const view = new DataView(buf); + const writeStr = (off, str) => { for (let i = 0; i < str.length; i++) view.setUint8(off + i, str.charCodeAt(i)); }; + const sampleRate = 16000, bitsPerSample = 16, numChannels = 1; + const byteRate = sampleRate * numChannels * bitsPerSample / 8; + const blockAlign = numChannels * bitsPerSample / 8; + writeStr(0, 'RIFF'); + view.setUint32(4, 36 + totalLen, true); + writeStr(8, 'WAVE'); + writeStr(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitsPerSample, true); + writeStr(36, 'data'); + view.setUint32(40, totalLen, true); + let offset = 44; + for (const chunk of pcmChunks) { + const src = new Uint8Array(chunk instanceof ArrayBuffer ? chunk : chunk.buffer || chunk); + new Uint8Array(buf, offset).set(src); + offset += src.byteLength; + } + return new Blob([buf], { type: 'audio/wav' }); +} + +elBtnExportWav.addEventListener('click', () => { + if (audioChunks.length === 0) { showToast('暂无音频数据', true); return; } + const blob = buildWavBlob(audioChunks); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = (currentVoiceId || 'audio') + '.wav'; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); + showToast('WAV 已导出'); +}); + +// ===== 辅助函数 ===== +function formatTime(ms) { + const totalSec = Math.floor(ms / 1000); + const min = String(Math.floor(totalSec / 60)).padStart(2, '0'); + const sec = String(totalSec % 60).padStart(2, '0'); + return `${min}:${sec}`; +} + +function formatTimeRange(startMs, endMs) { + return `${formatTime(startMs)} - ${formatTime(endMs)}`; +} + +function setStatus(state, text) { + elStatusDot.className = 'status-dot ' + state; + elStatusText.textContent = text; +} + +// ===== 渲染字幕（关闭说话人分离） ===== +// 每个 sentence_id 对应一个独立气泡： +// - 中间态（sentence_type===0）：实时更新该气泡文本，末尾加 "..." 表示未定。 +// - 稳态（sentence_type===1）：去掉 "..." 并定格，下一个 sentence_id 自动创建新气泡。 +function renderSubtitle(sentence) { + const id = 'subtitle-' + sentence.sentence_id; + const isInterim = sentence.sentence_type === 0; + let entry = sentenceMap[id]; + + if (!entry) { + const el = document.createElement('div'); + el.className = 'subtitle-item subtitle-interim'; + el.id = id; + const timeSpan = document.createElement('span'); + timeSpan.className = 'subtitle-time'; + el.appendChild(timeSpan); + const textSpan = document.createElement('span'); + textSpan.className = 'subtitle-text'; + el.appendChild(textSpan); + elResultArea.appendChild(el); + entry = { el: el }; + sentenceMap[id] = entry; + } + + const el = entry.el; + el.querySelector('.subtitle-time').textContent = formatTimeRange(sentence.start_time, sentence.end_time); + const textSpan = el.querySelector('.subtitle-text'); + if (isInterim) { + textSpan.textContent = sentence.sentence + ' ...'; + textSpan.className = 'subtitle-text interim'; + } else { + textSpan.textContent = sentence.sentence; + textSpan.className = 'subtitle-text'; + el.className = 'subtitle-item'; // 去掉 interim 类名，定格 + } + elResultArea.scrollTop = elResultArea.scrollHeight; +} + +// ===== 渲染说话人气泡 ===== +// 未确认片段独立展示，不能临时塞进上一位说话人的气泡。 +function renderBubble(sentence) { + const id = 'sent-' + sentence.sentence_id; + const speakerId = Number(sentence.speaker_id); + const trusted = Number.isInteger(speakerId) && speakerId >= 0 + && ['fresh', 'confirmed'].includes(sentence.speaker_evidence); + const isInterim = sentence.sentence_type === 0; + let entry = sentenceMap[id]; + if (!entry) { + const el = document.createElement('div'); + el.id = id; + const wrapper = document.createElement('div'); + wrapper.className = 'bubble-wrapper'; + const header = document.createElement('div'); + header.className = 'bubble-header'; + for (const name of ['speaker-badge', 'speaker-name', 'bubble-time']) { + const span = document.createElement('span'); + span.className = name; + header.appendChild(span); + } + const body = document.createElement('div'); + body.className = 'bubble-body'; + wrapper.append(header, body); + el.appendChild(wrapper); + elResultArea.appendChild(el); + entry = { el }; + sentenceMap[id] = entry; + } + if (trusted && !(speakerId in speakerOrderMap)) speakerOrderMap[speakerId] = speakerOrderCounter++; + const order = trusted ? speakerOrderMap[speakerId] : 0; + const color = order % SPEAKER_COLORS.length; + const el = entry.el; + el.className = trusted ? `bubble-row speaker-${order % 2 ? 'right' : 'left'} speaker-${color}` + : 'bubble-row speaker-left speaker-unknown'; + el.querySelector('.speaker-badge').className = 'speaker-badge speaker-color-' + (trusted ? color : 'unknown'); + // 未获得当前片段的可靠声纹证据时，标题保持简短；详细原因放到悬停提示， + // 这样不会把“有效语音不足……”等内部诊断信息挤进说话人名称区域。 + const speakerName = el.querySelector('.speaker-name'); + speakerName.textContent = trusted + ? (sentence.speaker_name || `说话人 ${speakerId + 1}`) + : '未知说话人'; + speakerName.title = trusted ? '' : (sentence.speaker_reason || '未匹配到说话人'); + el.querySelector('.bubble-time').textContent = formatTimeRange(sentence.start_time, sentence.end_time); + const body = el.querySelector('.bubble-body'); + body.textContent = sentence.sentence + (isInterim ? ' ...' : ''); + body.className = 'bubble-body' + (isInterim ? ' interim' : ''); + elResultArea.scrollTop = elResultArea.scrollHeight; +} + +// 按完整快照重建相邻块；序号防止两个后台 worker 的旧快照覆盖新状态。 +function renderDisplayState(msg, useSpeaker) { + if (msg.revision != null && msg.revision <= displayRevision) return; + if (msg.revision != null) displayRevision = msg.revision; + elResultArea.replaceChildren(); + sentenceMap = {}; + const raw = msg.raw_segments || msg.sentences || []; + if (useSpeaker) { + for (const block of msg.display_blocks || []) renderBubble({ ...block, sentence_id: block.block_id }); + const confirmed = raw.filter(s => s.speaker_status === 'confirmed').length; + const failed = raw.filter(s => ['service_error', 'service_unavailable', 'no_embedding', 'evidence_rejected'].includes(s.speaker_status)).length; + elSpeakerStatus.textContent = `说话人：已确认 ${confirmed} / ${raw.length} 段` + (failed ? `，${failed} 段未识别成功（原因见气泡及日志）` : ''); + } else { + raw.forEach(renderSubtitle); + elSpeakerStatus.textContent = '说话人分离已关闭'; + } +} + +elBtnStart.addEventListener('click', () => { + if (inputMode === 'file' && !selectedFile) return; + startRecognition(); +}); + +async function startRecognition() { + if (ws) return; + if (inputMode === 'file' && selectedFile) { + const ext = getFileExt(selectedFile.name); + if (!STREAMABLE_AUDIO_EXTENSIONS.has(ext)) { + showToast('实时流式测试只支持 PCM 或 WAV，请转换后再试', true); + return; + } + } + + elResultArea.innerHTML = ''; + elLogArea.innerHTML = ''; + sentenceMap = {}; + speakerOrderMap = {}; + speakerOrderCounter = 0; + displayStateSupported = false; + displayRevision = -1; + elSpeakerStatus.textContent = '正在检查说话人服务…'; + audioChunks = []; + elBtnExportWav.disabled = true; + elResultPlaceholder?.remove(); + elResultMeta.style.display = 'flex'; + elVoiceIdDisplay.textContent = '—'; + setStatus('connecting', '连接中...'); + + elBtnStart.disabled = true; + elBtnStop.disabled = false; + sending = true; + + const currentSession = ++sessionId; + const useSpeaker = elSpeakerDiarization.checked; + let receivedTerminal = false; + + // 构造 WebSocket 首条 start 消息。 + let voiceFormat = 0, fileName = '', speedFactor = 0; + if (inputMode === 'file') { + const ext = getFileExt(selectedFile.name); + voiceFormat = EXT_FORMAT_MAP[ext] || 0; + fileName = selectedFile.name; + speedFactor = Math.min(parseFloat(elSpeedSlider.value) || 1.0, MAX_SPEED); + } + + const startPayload = { + type: 'start', + model: elEngineModel.value, + model_service_url: elModelServiceUrl.value, + display_merge: elDisplayMerge.checked, + speaker_diarization: useSpeaker ? 1 : 0, + sentence_strategy: parseInt(elSentenceStrategy.value), + source: inputMode, + voice_format: voiceFormat, + file_name: fileName, + speed_factor: speedFactor + }; + + const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; + ws = new WebSocket(`${protocol}//${location.host}/ws`); + ws.binaryType = 'arraybuffer'; + + ws.onopen = () => { + if (currentSession !== sessionId) return; + ws.send(JSON.stringify(startPayload)); + // 在连接尚未建立时点击停止，也要在 start 后补发停止信号。 + if (!sending) ws.send(JSON.stringify({ type: 'stop' })); + }; + + ws.onmessage = (event) => { + if (currentSession !== sessionId) return; + const msg = JSON.parse(event.data); + if (msg.type === 'end' || msg.type === 'error') receivedTerminal = true; + if (msg.type !== 'sentences') { + console.log('[ws] type=' + msg.type, msg); + } + appendLog(msg); + handleServerMessage(msg, useSpeaker); + }; + + ws.onerror = (err) => { + if (currentSession !== sessionId) return; + console.error('WebSocket error:', err); + setStatus('error', '连接错误'); + stopMicCapture(); + resetControls(); + }; + + ws.onclose = () => { + if (currentSession !== sessionId) return; + stopMicCapture(); + if (!receivedTerminal) setStatus('error', '连接中断，最终识别结果可能尚未完成'); + if (audioChunks.length > 0) elBtnExportWav.disabled = false; + ws = null; + resetControls(); + }; +} + +function handleServerMessage(msg, useSpeaker) { + switch (msg.type) { + case 'voice_id': + if (msg.voice_id) { + currentVoiceId = msg.voice_id; + elVoiceIdDisplay.textContent = msg.voice_id; + } + break; + + case 'start': + displayStateSupported = Boolean(msg.display_state_supported); + currentVoiceId = msg.session_id; + elVoiceIdDisplay.textContent = currentVoiceId || '—'; + elSpeakerStatus.textContent = useSpeaker + ? `说话人服务：${msg.speaker_service_url || '未配置'}；片段结束后提取声纹` + : '说话人分离已关闭'; + if (!sending) break; + setStatus('running', '识别中...'); + if (inputMode === 'file') sendAudioFile(selectedFile).catch(handleInputError); + else startMicCapture().catch(handleInputError); + break; + + case 'display_state': + renderDisplayState(msg, useSpeaker); + break; + + case 'draining': + setStatus('running', msg.message || '等待最终识别结果…'); + break; + + case 'sentences': + if (displayStateSupported) break; + if (msg.sentences) { + msg.sentences.forEach(s => { + if (useSpeaker) renderBubble(s); + else renderSubtitle(s); + }); + } + break; + + case 'speaker_warning': + // ASR 仍可继续输出，但必须让测试人员立即知道说话人链路没有生效。 + elSpeakerStatus.textContent = '说话人服务异常：' + msg.message; + showToast('说话人服务异常，详见状态和片段原因', true); + break; + + case 'end': + if (msg.display_blocks) renderDisplayState(msg, useSpeaker); + setStatus('done', '识别完成'); + sending = false; + if (audioChunks.length > 0) elBtnExportWav.disabled = false; + resetControls(); + if (ws) { ws.close(); ws = null; } + break; + + case 'error': + setStatus('error', `错误: ${msg.message}`); + sending = false; + stopMicCapture(); + if (ws) { ws.close(); ws = null; } + resetControls(); + break; + } +} + +// ===== 停止识别 ===== +elBtnStop.addEventListener('click', () => stopRecognition()); + +function stopRecognition() { + sending = false; + stoppingByUser = true; + stopMicCapture(); + setStatus('running', '停止中...'); + elBtnStop.disabled = true; + + if (ws && ws.readyState === WebSocket.OPEN) { + try { ws.send(JSON.stringify({ type: 'stop' })); } catch (e) {} + } + + // 等待服务端排空 ASR/声纹队列后发送 end，不能用五秒计时器截断更新。 + +} + +function resetControls() { + sending = false; + stoppingByUser = false; + currentVoiceId = null; + if (inputMode === 'file') { + elBtnStart.disabled = !selectedFile; + } else { + elBtnStart.disabled = false; + } + elBtnStop.disabled = true; +} + +// ===== 发送音频文件 ===== +// 按 16KB 切片发送，并按照音频实际时长等待，确保文件模式也是真实的 +// 实时输入，而不是瞬间上传完整文件后再由服务端批量切片。 +const UPLOAD_CHUNK_SIZE = 16000; +async function sendAudioFile(file) { + const ownerSession = sessionId; + const buffer = await file.arrayBuffer(); + if (ownerSession !== sessionId || !sending) return; + const totalBytes = buffer.byteLength; + let offset = 0; + const ext = getFileExt(file.name); + const isPcm = (ext === 'pcm'); + let bytesPerSecond = 16000 * 2; + if (ext === 'wav' && totalBytes >= 44) { + const header = new DataView(buffer, 0, 44); + const byteRate = header.getUint32(28, true); + if (byteRate > 0) bytesPerSecond = byteRate; + } + const speedFactor = Math.max(parseFloat(elSpeedSlider.value) || 1.0, 0.1); + while (ownerSession === sessionId && offset < totalBytes && sending && ws && ws.readyState === WebSocket.OPEN) { + const end = Math.min(offset + UPLOAD_CHUNK_SIZE, totalBytes); + const chunk = buffer.slice(offset, end); + // 仅 PCM 数据可直接拼成 WAV 导出；当前实时模式不会接收压缩格式。 + if (isPcm) audioChunks.push(chunk.slice(0)); + ws.send(chunk); + offset = end; + const chunkDurationMs = (chunk.byteLength / bytesPerSecond) * 1000 / speedFactor; + await new Promise(r => setTimeout(r, Math.max(0, Math.round(chunkDurationMs)))); + } + if (ownerSession === sessionId && ws && ws.readyState === WebSocket.OPEN && sending) { + sending = false; + setStatus('running', '音频已发送，等待最终结果…'); + ws.send(JSON.stringify({ type: 'eof' })); + } +} + +// ===== 麦克风采集 ===== +async function startMicCapture() { + const ownerSession = sessionId; + let stream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true } + }); + } catch (err) { + if (ownerSession !== sessionId) return; + handleInputError(err); + console.error('getUserMedia error:', err); + setStatus('error', '无法获取麦克风权限'); + elMicStatus.textContent = '无法获取麦克风: ' + err.message; + resetControls(); + return; + } + + if (ownerSession !== sessionId || !sending) { + stream.getTracks().forEach(track => track.stop()); + return; + } + micStream = stream; + micAudioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 }); + const source = micAudioContext.createMediaStreamSource(micStream); + const processor = micAudioContext.createScriptProcessor(4096, 1, 1); + + processor.onaudioprocess = (e) => { + if (!sending || !ws || ws.readyState !== WebSocket.OPEN) return; + const float32 = e.inputBuffer.getChannelData(0); + const int16 = new Int16Array(float32.length); + for (let i = 0; i < float32.length; i++) { + let s = Math.max(-1, Math.min(1, float32[i])); + int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + ws.send(int16.buffer); + audioChunks.push(int16.buffer.slice(0)); + }; + + source.connect(processor); + processor.connect(micAudioContext.destination); + micWorklet = { source, processor }; + + micStartTime = Date.now(); + elMicStatus.style.display = 'none'; + elMicTimer.style.display = ''; + micTimerInterval = setInterval(() => { + const elapsed = Math.floor((Date.now() - micStartTime) / 1000); + const mm = String(Math.floor(elapsed / 60)).padStart(2, '0'); + const ss = String(elapsed % 60).padStart(2, '0'); + elMicElapsed.textContent = `${mm}:${ss}`; + }, 200); +} + +function stopMicCapture() { + if (micWorklet) { + micWorklet.processor.disconnect(); + micWorklet.source.disconnect(); + micWorklet = null; + } + if (micAudioContext) { + micAudioContext.close().catch(() => {}); + micAudioContext = null; + } + if (micStream) { + micStream.getTracks().forEach(t => t.stop()); + micStream = null; + } + if (micTimerInterval) { + clearInterval(micTimerInterval); + micTimerInterval = null; + } + elMicTimer.style.display = 'none'; + elMicStatus.style.display = ''; + elMicStatus.textContent = '点击下方按钮开始录音'; + elMicElapsed.textContent = '00:00'; +} + +// 展示实际部署端点及模型，避免沿用旧 SDK 的无效引擎配置。 +fetch('/api/config').then(response => response.json()).then(config => { + if (!ws) { + elEngineModel.value = config.model; + elModelServiceUrl.value = config.model_service_url; + elSpeakerStatus.textContent = '说话人辅助服务：' + (config.speaker_service_url || '未配置'); + } +}).catch(error => { elSpeakerStatus.textContent = '读取服务配置失败：' + error.message; }); + +// 输入端失败必须释放空会话，避免用户再次开始时留下旧连接。 +function handleInputError(error) { + sending = false; + setStatus('error', error.message); + stopMicCapture(); + if (ws) { ws.close(); ws = null; } + resetControls(); +} diff --git a/realtime_websocket/static/index.html b/realtime_websocket/static/index.html new file mode 100644 index 0000000..d34384d --- /dev/null +++ b/realtime_websocket/static/index.html @@ -0,0 +1,119 @@ + + + + + + 语音识别体验 Demo + + + +
+ + + + +
+
+

识别结果

+

正在读取服务配置…

+ +
+
选择音频来源并点击"开始识别"
+
+
+
+
+

ASR 原始日志

+ +
+
+
+
+
+ + + + + diff --git a/realtime_websocket/static/style.css b/realtime_websocket/static/style.css new file mode 100644 index 0000000..66e0211 --- /dev/null +++ b/realtime_websocket/static/style.css @@ -0,0 +1,805 @@ +/* ===== Reset & Base ===== */ +*, +*::before, +*::after { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + background: #f0f2f5; + color: #1a1a2e; + line-height: 1.6; + min-height: 100vh; +} + +/* ===== Two-Column Layout ===== */ +.layout { + display: flex; + height: 100vh; + overflow: hidden; +} + +.panel-left { + width: 340px; + min-width: 300px; + flex-shrink: 0; + padding: 20px 16px; + overflow-y: auto; + border-right: 1px solid #e5e5e5; + background: #fff; +} + +.panel-right { + flex: 1; + padding: 20px 20px; + overflow: hidden; + display: flex; + flex-direction: column; +} + +/* ===== Header ===== */ +header { + text-align: center; + margin-bottom: 20px; +} + +header h1 { + font-size: 22px; + font-weight: 700; + color: #1a1a2e; +} + +.subtitle { + color: #666; + font-size: 13px; + margin-top: 2px; +} + +/* ===== Card ===== */ +.card { + background: #fff; + border-radius: 10px; + padding: 16px 18px; + margin-bottom: 14px; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.06); +} + +.panel-left .card { + background: #f9fafb; +} + +.card h2 { + font-size: 14px; + font-weight: 600; + color: #333; + margin-bottom: 12px; + padding-bottom: 6px; + border-bottom: 1px solid #eee; +} + +/* ===== Collapsible Card Header ===== */ +.card-header-collapsible { + display: flex; + align-items: center; + justify-content: space-between; + cursor: pointer; + user-select: none; + padding-bottom: 0; +} + +.card-header-collapsible h2 { + margin-bottom: 0; + padding-bottom: 0; + border-bottom: none; +} + +.collapse-arrow { + font-size: 11px; + color: #999; + transition: transform 0.2s; +} + +.collapse-arrow.expanded { + color: #4a7dff; +} + +.advanced-params-body { + margin-top: 12px; + padding-top: 10px; + border-top: 1px solid #eee; +} + +/* ===== Section Divider ===== */ +.section-divider { + border-top: 1px solid #eee; + margin: 12px 0; +} + +/* ===== Form (left panel) ===== */ +.form-stack { + display: flex; + flex-direction: column; + gap: 12px; +} + +.form-row { + display: flex; + gap: 16px; +} + +.form-row .form-group { + flex: 1; +} + +.form-row-3 { + display: flex; + gap: 10px; +} + +.form-row-3 .form-group { + flex: 1; + min-width: 0; +} + +.form-group label:first-child { + display: block; + font-size: 12px; + font-weight: 500; + color: #555; + margin-bottom: 4px; +} + +.hint { + color: #999; + font-weight: 400; +} + +select, +input[type="text"] { + width: 100%; + padding: 7px 10px; + border: 1px solid #d9d9d9; + border-radius: 6px; + font-size: 13px; + outline: none; + transition: border-color 0.2s; +} + +select:focus, +input[type="text"]:focus { + border-color: #4a7dff; + box-shadow: 0 0 0 2px rgba(74, 125, 255, 0.15); +} + +input[type="text"][readonly] { + background: #f7f8fa; + color: #666; + cursor: default; +} + +input[type="text"][readonly]:focus { + border-color: #d9d9d9; + box-shadow: none; +} + +/* ===== Toggle Switch ===== */ +.toggle { + display: inline-flex; + align-items: center; + cursor: pointer; + gap: 6px; + user-select: none; +} + +.toggle input { + display: none; +} + +.toggle-slider { + width: 36px; + height: 20px; + background: #ccc; + border-radius: 10px; + position: relative; + transition: background 0.2s; +} + +.toggle-slider::after { + content: ""; + position: absolute; + width: 16px; + height: 16px; + background: #fff; + border-radius: 50%; + top: 2px; + left: 2px; + transition: transform 0.2s; +} + +.toggle input:checked + .toggle-slider { + background: #4a7dff; +} + +.toggle input:checked + .toggle-slider::after { + transform: translateX(16px); +} + +.toggle-label { + font-size: 12px; + color: #666; +} + +/* ===== Audio Input (left panel) ===== */ +.audio-input-stack { + display: flex; + flex-direction: column; + gap: 10px; +} + +/* Input mode tabs */ +.input-mode-tabs { + display: flex; + gap: 0; + border: 1px solid #d9d9d9; + border-radius: 6px; + overflow: hidden; +} + +.mode-tab { + flex: 1; + padding: 6px 0; + border: none; + background: #fff; + font-size: 12px; + font-weight: 500; + color: #666; + cursor: pointer; + transition: all 0.2s; + text-align: center; +} + +.mode-tab + .mode-tab { + border-left: 1px solid #d9d9d9; +} + +.mode-tab.active { + background: #4a7dff; + color: #fff; +} + +.mode-tab:hover:not(.active) { + background: #f0f5ff; +} + +/* File select */ +.file-select { + display: flex; + align-items: center; + gap: 10px; +} + +.file-info { + font-size: 12px; + color: #888; + word-break: break-all; +} + +.file-info.has-file { + color: #333; + font-weight: 500; +} + +/* Audio meta info */ +.audio-meta { + background: #f7f8fa; + border-radius: 6px; + padding: 8px 10px; + font-size: 12px; +} + +.audio-meta-row { + display: flex; + justify-content: space-between; + padding: 2px 0; +} + +.audio-meta-row .meta-k { + color: #888; +} + +.audio-meta-row .meta-v { + color: #333; + font-family: "SF Mono", Menlo, monospace; + font-size: 11px; +} + +/* Speed control */ +.speed-control { + display: flex; + flex-direction: column; + gap: 4px; +} + +.speed-label { + font-size: 12px; + font-weight: 500; + color: #555; +} + +.speed-label span { + color: #4a7dff; + font-weight: 600; +} + +#speedSlider { + width: 100%; + height: 4px; + -webkit-appearance: none; + appearance: none; + background: #e5e5e5; + border-radius: 2px; + outline: none; +} + +#speedSlider::-webkit-slider-thumb { + -webkit-appearance: none; + width: 16px; + height: 16px; + border-radius: 50%; + background: #4a7dff; + cursor: pointer; +} + +/* Microphone panel */ +.mic-status { + text-align: center; + color: #888; + font-size: 13px; + padding: 8px 0; +} + +.mic-timer { + text-align: center; + font-size: 14px; + font-weight: 600; + color: #ff4d4f; + padding: 8px 0; + animation: pulse 1s infinite; +} + +.mic-timer span { + font-family: "SF Mono", Menlo, monospace; + margin-left: 4px; +} + +.controls { + display: flex; + gap: 8px; +} + +/* ===== Buttons ===== */ +.btn { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 7px 16px; + border: none; + border-radius: 6px; + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: all 0.2s; + white-space: nowrap; +} + +.btn:disabled { + opacity: 0.45; + cursor: not-allowed; +} + +.btn-primary { + background: #4a7dff; + color: #fff; +} + +.btn-primary:hover:not(:disabled) { + background: #3a6ae8; +} + +.btn-danger { + background: #ff4d4f; + color: #fff; +} + +.btn-danger:hover:not(:disabled) { + background: #e63e40; +} + +.btn-outline { + background: #fff; + color: #4a7dff; + border: 1px solid #4a7dff; +} + +.btn-outline:hover { + background: #f0f5ff; +} + +.btn-icon { + background: none; + border: none; + cursor: pointer; + font-size: 14px; + padding: 2px 4px; + border-radius: 4px; + transition: background 0.15s; +} + +.btn-icon:hover { + background: #f0f0f0; +} + +.btn-icon.copy-success { + background: #e6f7e9; +} + +/* Export WAV button */ +.btn-export { + margin-left: 4px; + font-size: 11px !important; + padding: 2px 8px !important; +} + +.btn-export:disabled { + opacity: 0.35; + cursor: not-allowed; +} + +/* ===== Copy Toast ===== */ +.copy-toast { + position: fixed; + top: 20px; + left: 50%; + transform: translateX(-50%) translateY(-10px); + background: #333; + color: #fff; + padding: 6px 18px; + border-radius: 6px; + font-size: 13px; + font-weight: 500; + z-index: 9999; + opacity: 0; + transition: opacity 0.25s, transform 0.25s; + pointer-events: none; + white-space: nowrap; + box-shadow: 0 4px 12px rgba(0,0,0,0.15); +} + +.copy-toast.visible { + opacity: 1; + transform: translateX(-50%) translateY(0); +} + +.copy-toast-error { + background: #ff4d4f; +} + +/* ===== Result Section (right panel) ===== */ +.result-card { + flex: 1; + min-height: 0; + display: flex; + flex-direction: column; + overflow: hidden; + background: #fff; +} + +.result-meta { + display: flex; + align-items: center; + gap: 20px; + margin-bottom: 10px; + padding: 6px 10px; + background: #f7f8fa; + border-radius: 6px; + font-size: 12px; + flex-shrink: 0; +} + +.meta-item { + display: flex; + align-items: center; + gap: 5px; +} + +.meta-label { + color: #888; +} + +.meta-value { + color: #333; + font-family: "SF Mono", Menlo, monospace; + font-size: 11px; +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #ccc; +} + +.status-dot.connecting { + background: #faad14; + animation: pulse 1s infinite; +} + +.status-dot.running { + background: #52c41a; + animation: pulse 1s infinite; +} + +.status-dot.done { + background: #4a7dff; +} + +.status-dot.error { + background: #ff4d4f; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.4; } +} + +.result-area { + flex: 1; + overflow-y: auto; + padding: 4px 0; +} + +.placeholder { + text-align: center; + color: #bbb; + padding: 60px 0; + font-size: 14px; +} + +/* ===== Subtitle Mode (no speaker diarization) ===== */ +.subtitle-item { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 5px 8px; + border-radius: 6px; + margin-bottom: 3px; + transition: background 0.15s; +} + +.subtitle-item:hover { + background: #f7f8fa; +} + +.subtitle-time { + font-size: 11px; + color: #999; + font-family: "SF Mono", Menlo, monospace; + white-space: nowrap; + min-width: 100px; + padding-top: 2px; +} + +.subtitle-text { + font-size: 14px; + color: #1a1a2e; + word-break: break-word; + transition: color 0.3s ease; +} + +.subtitle-text.interim { + color: #999; + font-style: italic; +} + +.subtitle-text.final { + color: #000; + font-style: normal; + font-weight: 500; +} + +.subtitle-item.subtitle-interim { + opacity: 0.8; +} + +/* ===== Bubble Mode (speaker diarization) ===== */ +.bubble-row { + display: flex; + margin-bottom: 10px; +} + +.bubble-row.speaker-left { + justify-content: flex-start; +} + +.bubble-row.speaker-right { + justify-content: flex-end; +} + +.bubble-row.speaker-unknown .bubble-body { + background: #f0f0f0; + color: #999; + font-style: italic; + border-radius: 12px; + border-top-left-radius: 4px; +} + +.speaker-badge.speaker-color-unknown { + background-color: #ccc; +} + +.bubble-wrapper { + max-width: 80%; +} + +.bubble-header { + font-size: 11px; + color: #888; + margin-bottom: 3px; + display: flex; + align-items: center; + gap: 5px; +} + +.speaker-right .bubble-header { + justify-content: flex-end; +} + +.speaker-badge { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 50%; +} + +.bubble-body { + padding: 9px 13px; + border-radius: 12px; + font-size: 14px; + line-height: 1.5; + word-break: break-word; +} + +.speaker-left .bubble-body { + background: #e8f0fe; + color: #1a1a2e; + border-top-left-radius: 4px; +} + +.speaker-right .bubble-body { + background: #e6f7e9; + color: #1a1a2e; + border-top-right-radius: 4px; + text-align: right; +} + +.bubble-body.interim { + opacity: 0.6; + font-style: italic; +} + +/* Pending text (speaker_id=-1) appended to confirmed bubble */ +.pending-text { + display: inline; + color: #aaa; + font-style: italic; + opacity: 0.7; + margin-left: 2px; + transition: opacity 0.3s ease; +} + +.pending-text::before { + content: ' '; +} + +.bubble-time { + font-size: 10px; + color: #bbb; + font-family: "SF Mono", Menlo, monospace; +} + +/* ===== Speaker Colors ===== */ +.speaker-color-0 { background-color: #4a7dff; } +.speaker-color-1 { background-color: #52c41a; } +.speaker-color-2 { background-color: #faad14; } +.speaker-color-3 { background-color: #ff4d4f; } +.speaker-color-4 { background-color: #9254de; } +.speaker-color-5 { background-color: #13c2c2; } + +.bubble-row.speaker-2 .bubble-body { background: #fff7e6; } +.bubble-row.speaker-3 .bubble-body { background: #fff1f0; } +.bubble-row.speaker-4 .bubble-body { background: #f9f0ff; } +.bubble-row.speaker-5 .bubble-body { background: #e6fffb; } + +/* ===== Responsive ===== */ +@media (max-width: 768px) { + .layout { + flex-direction: column; + height: auto; + } + .panel-left { + width: 100%; + min-width: unset; + border-right: none; + border-bottom: 1px solid #e5e5e5; + max-height: 40vh; + } + .panel-right { + min-height: 60vh; + } +} + +/* ===== Log Panel ===== */ +.log-card { + height: 240px; + min-height: 180px; + flex-shrink: 0; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.log-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 8px; + padding-bottom: 6px; + border-bottom: 1px solid #eee; +} + +.log-header h2 { + margin-bottom: 0; + padding-bottom: 0; + border-bottom: none; +} + +.btn-sm { + padding: 3px 10px; + font-size: 11px; +} + +.log-area { + flex: 1; + overflow-y: auto; + background: #1e1e2e; + border-radius: 6px; + padding: 8px 10px; + font-family: "SF Mono", Menlo, Consolas, monospace; + font-size: 11px; + line-height: 1.5; + color: #cdd6f4; +} + +.log-entry { + padding: 2px 0; + border-bottom: 1px solid rgba(255,255,255,0.05); + word-break: break-all; +} + +.log-time { + color: #6c7086; + margin-right: 6px; +} + +.log-type-start { color: #a6e3a1; } +.log-type-voice_id { color: #cba6f7; } +.log-type-sentences { color: #89b4fa; } +.log-type-result { color: #89b4fa; } +.log-type-end { color: #f9e2af; } +.log-type-error { color: #f38ba8; } + + + diff --git a/realtime_websocket/tests/test_frontend.cjs b/realtime_websocket/tests/test_frontend.cjs new file mode 100644 index 0000000..ea53491 --- /dev/null +++ b/realtime_websocket/tests/test_frontend.cjs @@ -0,0 +1,102 @@ +// 使用 Node 内置测试器和最小 DOM 验证真实页面脚本，无需安装浏览器或 GPU。 +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +class Element { + constructor() { + this.childNodes = []; + this.className = ''; + this.style = {}; + this.textContent = ''; + this.classList = { add() {}, remove() {}, toggle() {} }; + } + append(...nodes) { nodes.forEach(node => this.appendChild(node)); } + appendChild(node) { node.parent = this; this.childNodes.push(node); } + replaceChildren() { this.childNodes = []; } + addEventListener() {} + remove() { if (this.parent) this.parent.childNodes = this.parent.childNodes.filter(n => n !== this); } + querySelector(selector) { + const name = selector.slice(1); + for (const child of this.childNodes) { + if (child.className.split(' ').includes(name)) return child; + const nested = child.querySelector(selector); + if (nested) return nested; + } + return null; + } + get firstChild() { return this.childNodes[0]; } +} + +// 每个测试使用独立页面上下文，不共享会话状态。 +function page() { + const elements = new Map(); + const timers = []; + const context = vm.createContext({ + document: { + getElementById(id) { if (!elements.has(id)) elements.set(id, new Element()); return elements.get(id); }, + createElement() { return new Element(); }, + body: new Element(), + }, + console, + setTimeout(fn, ms) { timers.push({ fn, ms }); }, + clearInterval() {}, + WebSocket: { OPEN: 1 }, + fetch: async () => ({ json: async () => ({ model: 'test', model_service_url: 'http://fake/v1' }) }), + }); + vm.runInContext(fs.readFileSync(path.join(__dirname, '../static/app.js'), 'utf8'), context); + return { context, elements, timers }; +} + +const block = (id, speaker, text = 'text', reason = '') => ({ + block_id: `block-${id}`, sentence: text, sentence_type: 1, start_time: id * 1000, end_time: (id + 1) * 1000, + speaker_id: speaker, speaker_name: speaker < 0 ? '' : `Person ${speaker}`, + speaker_evidence: speaker < 0 ? 'pending' : 'confirmed', speaker_reason: reason, +}); + +test('display snapshots update and merge pending rows; stale snapshots are ignored', () => { + const { context, elements } = page(); + context.renderDisplayState({ revision: 1, display_blocks: [block(0, -1), block(1, -1)] }, true); + assert.equal(elements.get('resultArea').childNodes.length, 2); + context.renderDisplayState({ revision: 2, display_blocks: [block(0, 0, 'A B')] }, true); + assert.equal(elements.get('resultArea').childNodes.length, 1); + assert.equal(elements.get('resultArea').querySelector('.speaker-name').textContent, 'Person 0'); + context.renderDisplayState({ revision: 1, display_blocks: [block(0, -1), block(1, -1)] }, true); + assert.equal(elements.get('resultArea').childNodes.length, 1); +}); + +test('A B A retains time order and an unknown interruption has its own bubble', () => { + const { context, elements } = page(); + context.renderDisplayState({ revision: 1, display_blocks: [block(0, 0), block(1, 1), block(2, 0), block(3, -1, 'short', '音频不足')] }, true); + const rows = elements.get('resultArea').childNodes; + assert.deepEqual(rows.map(row => row.querySelector('.speaker-name').textContent), ['Person 0', 'Person 1', 'Person 0', '未知说话人']); + assert.equal(rows[3].querySelector('.speaker-name').title, '音频不足'); +}); + +test('sentences do not duplicate display_state and end uses the final snapshot', () => { + const { context, elements } = page(); + vm.runInContext('displayStateSupported = true', context); + context.handleServerMessage({ type: 'sentences', sentences: [{ ...block(0, -1), sentence_id: 0 }] }, true); + assert.equal(elements.get('resultArea').childNodes.length, 0); + context.handleServerMessage({ type: 'end', display_blocks: [block(0, 0)], sentences: [] }, true); + assert.equal(elements.get('resultArea').childNodes.length, 1); +}); + +test('stop sends a control message without scheduling a forced close', () => { + const { context, timers } = page(); + vm.runInContext('var sent = []; ws = { readyState: 1, send(message) { sent.push(JSON.parse(message)); } }; sending = true', context); + context.stopRecognition(); + assert.equal(vm.runInContext('sent[0].type', context), 'stop'); + assert.equal(timers.length, 0); + assert.equal(vm.runInContext('ws !== null', context), true); +}); + +test('raw logs render untrusted transcript as text', () => { + const { context, elements } = page(); + context.appendLog({ type: 'sentences', text: '' }); + const entry = elements.get('logArea').childNodes[0]; + assert.equal(entry.innerHTML, undefined); + assert.ok(entry.childNodes[1].textContent.includes(' None: + wav_bytes = pcm16_to_wav(b"\x00\x00" * 160) + with wave.open(BytesIO(wav_bytes), "rb") as wav_file: + self.assertEqual(wav_file.getframerate(), 16000) + self.assertEqual(wav_file.getnchannels(), 1) + self.assertEqual(wav_to_pcm16(wav_bytes), b"\x00\x00" * 160) + + def test_compressed_partial_is_deferred(self) -> None: + self.assertIsNone(prepare_audio_request(b"partial", "file", "sample.mp3", partial=True)) + prepared = prepare_audio_request(b"complete", "file", "sample.mp3", partial=False) + self.assertEqual(prepared[1], "sample.mp3") + + def test_auxiliary_config_is_independent_from_vllm(self) -> None: + service = AuxiliaryModelService(AuxiliaryServiceConfig()) + self.assertEqual(service.config.base_url, "http://127.0.0.1:8010") + + +if __name__ == "__main__": + unittest.main() diff --git a/realtime_websocket/tests/test_server.py b/realtime_websocket/tests/test_server.py new file mode 100644 index 0000000..a5885b6 --- /dev/null +++ b/realtime_websocket/tests/test_server.py @@ -0,0 +1,388 @@ +"""使用模拟 VLLM 适配器验证本地 WebSocket 流程。""" + +from __future__ import annotations + +from types import SimpleNamespace +import asyncio +import io +import wave +import os +from unittest.mock import AsyncMock, patch + +from aiohttp import web +from aiohttp.test_utils import AioHTTPTestCase + +from server import ( + AUXILIARY_SERVICE_KEY, + MODEL_SERVICE_KEY, + config_handler, + deployment_model_name, + validate_model_service_url, + websocket_handler, +) + + +class FakeModelService: + native_partial_supported = False + config = SimpleNamespace(base_url="http://fake/v1", model="fake-model") + + async def transcribe(self, audio_bytes: bytes, source: str, file_name: str, partial: bool) -> str: + return "partial text" if partial else "final text" + + +class FakeAuxiliaryService: + """返回固定时间段的聚类服务，用于验证 final 后的同句 speaker 更新。""" + + def __init__(self) -> None: + self.calls = 0 + + async def health(self) -> dict[str, object]: + """模拟辅助模型服务已完成预加载。""" + return {"ready": True, "speaker_embedding_ready": True} + + async def resolve_speaker( + self, + audio_bytes: bytes, + session_id: str, + start_time_ms: float, + end_time_ms: float, + ) -> dict[str, object]: + _ = (audio_bytes, session_id, start_time_ms, end_time_ms) + speaker_id = [0, 1, 0][min(self.calls, 2)] + self.calls += 1 + return { + "speaker_id": speaker_id, + "speaker_name": f"说话人 {speaker_id + 1}", + "speaker_evidence": "fresh", + "speaker_confidence": 0.9, + "speaker_strategy": "online_embedding_cluster", + } + + async def reset_speaker_session(self, session_id: str) -> None: + _ = session_id + + +class UnhealthyAuxiliaryService(FakeAuxiliaryService): + """模拟端口可访问但辅助模型尚未就绪的服务。""" + + async def health(self) -> dict[str, object]: + return {"ready": False, "speaker_embedding_ready": False} + + +class WebSocketFlowTests(AioHTTPTestCase): + async def collect(self, ws, until="end"): + """限定等待时间，回归测试中的队列卡死必须表现为失败。""" + events = [] + async with asyncio.timeout(10): + while True: + event = await ws.receive_json() + events.append(event) + if event["type"] == until: + return events + + def get_app(self) -> web.Application: + app = web.Application() + app[MODEL_SERVICE_KEY] = FakeModelService() + app[AUXILIARY_SERVICE_KEY] = FakeAuxiliaryService() + app.router.add_get("/api/config", config_handler) + app.router.add_get("/ws", websocket_handler) + return app + + def test_vllm_url_validation(self) -> None: + self.assertEqual(validate_model_service_url(" http://asr.local/v1/ "), "http://asr.local/v1") + with self.assertRaises(ValueError): + validate_model_service_url("asr.local:8000/v1") + with self.assertRaises(ValueError): + validate_model_service_url("http://user:password@asr.local/v1") + + def test_deployment_alias_and_served_name_match_vllm(self): + """WebSocket 不能把下载别名直接当成 vLLM 公开模型名。""" + with patch.dict(os.environ, {"QWEN3_ASR_MODEL": "0.6b"}, clear=True): + self.assertEqual(deployment_model_name(), "Qwen/Qwen3-ASR-0.6B") + with patch.dict(os.environ, {"QWEN3_ASR_MODEL": "0.6b", "VLLM_SERVED_MODEL_NAME": "custom-asr"}, clear=True): + self.assertEqual(deployment_model_name(), "custom-asr") + + async def test_empty_final_retracts_partial_instead_of_leaving_pending(self): + """最终没有识别文本时撤回临时内容，不留下永远等待声纹的行。""" + async def transcribe(*args, partial): + return "temporary" if partial else "" + self.app[MODEL_SERVICE_KEY].transcribe = transcribe + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "partial_interval_ms": 300}) + await ws.receive_json() + await ws.send_bytes(b"\xe8\x03" * 16000) + await ws.send_json({"type": "eof"}) + events = await self.collect(ws) + self.assertTrue(any(e["type"] == "sentences" for e in events)) + self.assertEqual(events[-1]["sentences"], []) + self.assertEqual(events[-1]["display_blocks"], []) + + async def test_compressed_file_is_rejected_for_streaming_validation(self) -> None: + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "source": "file", "file_name": "meeting.mp3"}) + error = await ws.receive_json() + self.assertEqual(error["type"], "error") + self.assertIn("PCM 或 WAV", error["message"]) + await ws.close() + + async def test_short_interruption_does_not_inherit_or_call_embedding(self): + """句尾静音不能凑够声纹时长，长段 A 后的短插话独立保持 pending。""" + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "source": "mic"}) + await ws.receive_json() + await ws.send_bytes(b"\xe8\x03" * 16000 + b"\x00\x00" * 12800 + b"\xe8\x03" * 8000 + b"\x00\x00" * 12800) + await ws.send_json({"type": "stop"}) + events = await self.collect(ws) + final = events[-1] + self.assertEqual(self.app[AUXILIARY_SERVICE_KEY].calls, 1) + self.assertEqual([s["speaker_id"] for s in final["sentences"]], [0, -1]) + self.assertEqual(final["sentences"][1]["speaker_status"], "insufficient_audio") + self.assertEqual(final["sentences"][0]["end_time"], 1000) + self.assertEqual(len(final["display_blocks"]), 2) + + async def test_short_speaker_gap_splits_turn_before_next_speaker(self): + """说话人模式在短交接停顿处切段，保持每段独立进入在线聚类。""" + ws = await self.client.ws_connect("/ws") + await ws.send_json( + { + "type": "start", + "source": "mic", + "speaker_diarization": 1, + "speaker_gap_ms": 400, + "partial_interval_ms": 1200, + } + ) + self.assertEqual((await ws.receive_json())["type"], "start") + + voiced = b"\xe8\x03" * 16000 # 每位说话人一秒有效语音 + handoff_gap = b"\x00\x00" * 6400 # 400ms，低于原始 800ms 切段阈值 + trailing_silence = b"\x00\x00" * 12800 + await ws.send_bytes(voiced + handoff_gap + voiced + trailing_silence) + await ws.send_json({"type": "eof"}) + + events: list[dict[str, object]] = [] + while True: + message = await ws.receive_json() + events.append(message) + if message["type"] == "end": + break + + final_sentences = [ + message["sentences"][0] + for message in events + if message["type"] == "sentences" and message["sentences"][0]["sentence_type"] == 1 + ] + latest_by_id = {int(item["sentence_id"]): item for item in final_sentences} + latest = [latest_by_id[index] for index in sorted(latest_by_id)] + self.assertEqual(len(latest), 2) + self.assertEqual([item["speaker_id"] for item in latest], [0, 1]) + self.assertEqual(self.app[AUXILIARY_SERVICE_KEY].calls, 2) + await ws.close() + + async def test_stop_waits_for_slow_speaker_and_includes_final_snapshot(self): + """在 end 前必须收到所有声纹结果，不能复现页面原先五秒断开的行为。""" + auxiliary = self.app[AUXILIARY_SERVICE_KEY] + original = auxiliary.resolve_speaker + async def delayed(*args): + await asyncio.sleep(5.1) + return await original(*args) + auxiliary.resolve_speaker = delayed + auxiliary.reset_speaker_session = AsyncMock() + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start"}) + start = await ws.receive_json() + await ws.send_bytes(b"\xe8\x03" * 16000) + await ws.send_json({"type": "stop"}) + events = await self.collect(ws) + self.assertEqual(events[-1]["sentences"][0]["speaker_id"], 0) + self.assertTrue(any(e["type"] == "draining" for e in events)) + self.assertTrue(any(e["type"] == "sentences" and e["sentences"][0]["speaker_status"] == "processing" for e in events)) + await ws.receive() # 等待服务端执行 finally 并关闭连接 + auxiliary.reset_speaker_session.assert_awaited_once_with(start["session_id"]) + + async def test_speaker_error_is_visible_on_segment_and_asr_finishes(self): + """声纹推理失败不能吞掉转写，且每条失败片段要携带诊断原因。""" + self.app[AUXILIARY_SERVICE_KEY].resolve_speaker = AsyncMock(side_effect=RuntimeError("embedding model missing")) + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start"}) + await ws.receive_json() + await ws.send_bytes(b"\xe8\x03" * 16000) + await ws.send_json({"type": "eof"}) + events = await self.collect(ws) + segment = events[-1]["sentences"][0] + self.assertEqual(segment["sentence"], "final text") + self.assertEqual(segment["speaker_status"], "service_error") + self.assertIn("embedding model missing", segment["speaker_reason"]) + self.assertTrue(any(e["type"] == "speaker_warning" for e in events)) + + async def test_asr_failure_is_reported_before_stop(self): + """音频 worker 抛异常时，接收任务应立即报告，不能等到客户端 stop。""" + self.app[MODEL_SERVICE_KEY].transcribe = AsyncMock(side_effect=RuntimeError("vllm unavailable")) + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "partial_interval_ms": 300}) + await ws.receive_json() + await ws.send_bytes(b"\xe8\x03" * 16000) + events = await self.collect(ws, until="error") + self.assertIn("vllm unavailable", events[-1]["message"]) + + async def test_unknown_speaker_response_is_diagnosable(self): + """旧服务只回标签、缺少 fresh/confidence 时应说明拒绝原因。""" + self.app[AUXILIARY_SERVICE_KEY].resolve_speaker = AsyncMock(return_value={"speaker_id": 0}) + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start"}) + await ws.receive_json() + await ws.send_bytes(b"\xe8\x03" * 16000) + await ws.send_json({"type": "eof"}) + events = await self.collect(ws) + self.assertEqual(events[-1]["sentences"][0]["speaker_status"], "evidence_rejected") + + async def test_abort_and_disconnect_release_cluster_state(self): + """清理不应只存在于成功 stop 的路径。""" + auxiliary = self.app[AUXILIARY_SERVICE_KEY] + auxiliary.reset_speaker_session = AsyncMock() + for abort in (True, False): + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start"}) + await ws.receive_json() + if abort: + await ws.send_json({"type": "abort"}) + await ws.receive() + else: + await ws.close() + async with asyncio.timeout(2): + while auxiliary.reset_speaker_session.await_count < 2: + await asyncio.sleep(0.01) + self.assertEqual(auxiliary.reset_speaker_session.await_count, 2) + + async def test_extended_wav_header_is_removed_before_asr(self): + """分片 RIFF/JUNK/fmt/data 头不能混入声纹和 ASR 的 PCM 数据。""" + output = io.BytesIO() + pcm = b"\xe8\x03" * 16000 + with wave.open(output, "wb") as wav: + wav.setparams((1, 2, 16000, 0, "NONE", "")) + wav.writeframes(pcm) + original = output.getvalue() + junk = b"JUNK\x04\x00\x00\x00test" + payload = b"RIFF" + (len(original) - 8 + len(junk)).to_bytes(4, "little") + original[8:12] + junk + original[12:] + transcribe = AsyncMock(return_value="wav text") + self.app[MODEL_SERVICE_KEY].transcribe = transcribe + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "source": "file", "file_name": "test.wav"}) + await ws.receive_json() + for offset in range(0, len(payload), 337): + await ws.send_bytes(payload[offset:offset + 337]) + await ws.send_json({"type": "eof"}) + events = await self.collect(ws) + self.assertEqual(transcribe.call_args.args[0], pcm) + self.assertEqual(events[-1]["sentences"][0]["end_time"], 1000) + + async def test_incompatible_wav_is_rejected_immediately(self): + """非 16kHz 单声道 WAV 不能被误解释为可识别的 PCM16。""" + output = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setparams((2, 2, 44100, 0, "NONE", "")) + wav.writeframes(b"\x00\x00" * 2000) + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "source": "file", "file_name": "bad.wav"}) + await ws.receive_json() + await ws.send_bytes(output.getvalue()) + events = await self.collect(ws, until="error") + self.assertIn("16kHz", events[-1]["message"]) + + async def test_frontend_can_read_default_vllm_config(self) -> None: + response = await self.client.get("/api/config") + self.assertEqual(response.status, 200) + self.assertEqual(await response.json(), {"model_service_url": "http://fake/v1", "model": "fake-model", "speaker_service_url": None}) + + async def test_partial_and_final_share_one_sentence_id(self) -> None: + ws = await self.client.ws_connect("/ws") + await ws.send_json( + { + "type": "start", + "source": "mic", + "speaker_diarization": 0, + "partial_interval_ms": 300, + "max_segment_sec": 12, + } + ) + start = await ws.receive_json() + self.assertEqual(start["type"], "start") + # 使用幅度足够的 PCM 语音帧，静音帧会被新切句器正确忽略。 + await ws.send_bytes(b"\xe8\x03" * 16000) + + messages = [] + while True: + message = await ws.receive_json() + messages.append(message) + if message["type"] == "sentences": + self.assertEqual(message["sentences"][0]["sentence_id"], 0) + if message["sentences"][0]["sentence_type"] == 0: + break + + await ws.send_json({"type": "eof"}) + while True: + message = await ws.receive_json() + messages.append(message) + if message["type"] == "end": + break + + sentence_events = [message for message in messages if message["type"] == "sentences"] + self.assertGreaterEqual(len(sentence_events), 2) + self.assertTrue(all(event["sentences"][0]["sentence_id"] == 0 for event in sentence_events)) + self.assertTrue(all(event["sentences"][0]["sentence_type"] == 0 for event in sentence_events[:-1])) + self.assertEqual(sentence_events[-1]["sentences"][0]["sentence_type"], 1) + self.assertEqual(sentence_events[-1]["sentences"][0]["sentence"], "final text") + await ws.close() + + async def test_speaker_health_failure_is_reported_without_blocking_asr(self) -> None: + """辅助模型未就绪时先报告告警，同时保留 ASR 会话能力。""" + self.app[AUXILIARY_SERVICE_KEY] = UnhealthyAuxiliaryService() + ws = await self.client.ws_connect("/ws") + await ws.send_json({"type": "start", "source": "mic", "speaker_diarization": 1}) + + start = await ws.receive_json() + warning = await ws.receive_json() + + self.assertEqual(start["type"], "start") + self.assertFalse(start["speaker_service_health"]["ready"]) + self.assertFalse(start["speaker_gap_enabled"]) + self.assertEqual(warning["type"], "speaker_warning") + self.assertIn("未就绪", warning["message"]) + await ws.close() + + async def test_vad_split_and_speaker_update(self) -> None: + ws = await self.client.ws_connect("/ws") + await ws.send_json( + { + "type": "start", + "source": "mic", + "speaker_diarization": 1, + "partial_interval_ms": 1200, + } + ) + self.assertEqual((await ws.receive_json())["type"], "start") + + voiced = b"\xe8\x03" * 16000 # 每段一秒有效语音，满足独立声纹长度要求 + silence = b"\x00\x00" * 12800 # 0.8 秒静音，触发当前 turn 提交 + await ws.send_bytes(voiced + silence + voiced + silence + voiced) + await ws.send_json({"type": "eof"}) + + events: list[dict[str, object]] = [] + while True: + message = await ws.receive_json() + events.append(message) + if message["type"] == "end": + break + + sentence_events = [message for message in events if message["type"] == "sentences"] + final_sentences = [ + message["sentences"][0] + for message in sentence_events + if message["sentences"][0]["sentence_type"] == 1 + ] + latest_by_id = {int(item["sentence_id"]): item for item in final_sentences} + self.assertGreaterEqual(len(latest_by_id), 2) + latest = [latest_by_id[index] for index in sorted(latest_by_id)[-3:]] + self.assertEqual([item["speaker_id"] for item in latest], [0, 1, 0]) + self.assertTrue(all(item["speaker_evidence"] == "fresh" for item in latest)) + await ws.close() diff --git a/realtime_websocket/tests/test_speaker_assembler.py b/realtime_websocket/tests/test_speaker_assembler.py new file mode 100644 index 0000000..d4fd1fa --- /dev/null +++ b/realtime_websocket/tests/test_speaker_assembler.py @@ -0,0 +1,121 @@ +"""片段幂等更新和说话人安全规则的回归测试。""" + +from __future__ import annotations + +import unittest + +from speaker_assembler import SegmentAssembler + + +class SegmentAssemblerTests(unittest.TestCase): + def test_late_speaker_updates_merge_and_split_existing_blocks(self): + """模拟 final 先到、声纹后到以及身份修正，按当前完整状态重新分块。""" + assembler = SegmentAssembler() + for sid in range(3): + assembler.apply_sentence({"sentence_id": sid, "sentence": str(sid), "sentence_type": 1, + "start_time": sid * 2000, "end_time": sid * 2000 + 1000}) + for sid in range(3): + assembler.apply_speaker_update({"sentence_id": sid, "speaker_id": 0, "speaker_name": "A", + "speaker_evidence": "fresh", "speaker_confidence": 0.9}) + self.assertEqual(assembler.display_blocks()[0]["segment_ids"], [0, 1, 2]) + assembler.apply_speaker_update({"sentence_id": 1, "speaker_id": 1, "speaker_name": "B", + "speaker_evidence": "fresh", "speaker_confidence": 0.9}) + self.assertEqual([b["speaker_id"] for b in assembler.display_blocks()], [0, 1, 0]) + self.assertEqual(len(assembler.display_blocks(False)), 3) + + def test_speaker_update_cannot_inherit_or_change_text_and_timestamps(self): + """异步更新同样要拒绝继承证据，且模型响应不能改写原始转写。""" + assembler = SegmentAssembler() + assembler.apply_sentence({"sentence_id": 0, "sentence": "原文", "start_time": 0, "end_time": 500}) + updated = assembler.apply_speaker_update({ + "sentence_id": 0, "sentence": "错误原文", "start_time": 9000, + "speaker_id": 7, "speaker_name": "前一位", "user_id": "person-a", + "speaker_strategy": "short_attach", "speaker_evidence": "confirmed", "speaker_confidence": 0.99, + "_embedding": [1, 0], + }) + self.assertEqual(updated["speaker_id"], -1) + self.assertEqual(updated["sentence"], "原文") + self.assertEqual(updated["start_time"], 0) + self.assertNotIn("user_id", updated) + self.assertNotIn("_embedding", updated) + + def test_nonfinite_or_weak_confidence_is_not_trusted(self): + """NaN/Infinity 和不充分证据不得进入可信展示块。""" + for confidence in (float("nan"), float("inf"), 0.59): + assembler = SegmentAssembler() + assembler.apply_sentence({"sentence_id": 0, "sentence": "test"}) + result = assembler.apply_speaker_update({"sentence_id": 0, "speaker_id": 0, + "speaker_evidence": "fresh", "speaker_confidence": confidence}) + self.assertEqual(result["speaker_status"], "evidence_rejected") + self.assertEqual(assembler.display_blocks()[0]["speaker_id"], -1) + + def test_same_cluster_different_named_identity_does_not_merge(self): + """相同匿名簇 ID 不代表相同注册身份，弱匿名段不能挤入实名块。""" + assembler = SegmentAssembler() + for sid, user in enumerate(("A", "B", None)): + assembler.apply_sentence({"sentence_id": sid, "sentence": str(sid), "speaker_id": 0, + "speaker_evidence": "fresh", "speaker_confidence": 0.9, "user_id": user}) + self.assertEqual(len(assembler.display_blocks()), 3) + + def test_text_only_update_preserves_identity_and_final_is_not_rolled_back(self): + """文本与声纹独立更新；重发的旧 partial 不得回滚 final。""" + assembler = SegmentAssembler() + assembler.apply_sentence({"sentence_id": 0, "sentence": "final", "sentence_type": 1}) + assembler.apply_speaker_update({"sentence_id": 0, "speaker_id": 0, + "speaker_evidence": "fresh", "speaker_confidence": 0.9}) + assembler.apply_sentence({"sentence_id": 0, "sentence": "corrected", "sentence_type": 1}) + assembler.apply_sentence({"sentence_id": 0, "sentence": "old", "sentence_type": 0}) + self.assertEqual(assembler.raw_snapshot()[0]["sentence"], "corrected") + self.assertEqual(assembler.display_blocks()[0]["speaker_id"], 0) + + def test_same_sentence_id_is_updated_in_place(self) -> None: + assembler = SegmentAssembler() + assembler.apply_sentence({"sentence_id": 3, "sentence": "你好", "sentence_type": 0}) + final = assembler.apply_sentence({"sentence_id": 3, "sentence": "你好，今天开始。", "sentence_type": 1}) + + self.assertEqual(len(assembler.raw_snapshot()), 1) + self.assertEqual(final["sentence_type"], 1) + self.assertEqual(final["revision_count"], 1) + + def test_short_inherited_name_is_pending_and_embedding_is_dropped(self) -> None: + assembler = SegmentAssembler() + segment = assembler.apply_sentence( + { + "sentence_id": 1, + "sentence": "嗯", + "start_time": 0, + "end_time": 900, + "speaker_id": 7, + "speaker_name": "上一位", + "speaker_strategy": "short_attach", + "speaker_evidence": "confirmed", + "speaker_confidence": 0.99, + "_embedding": [1, 2, 3], + } + ) + + self.assertEqual(segment["speaker_id"], -1) + self.assertEqual(segment["speaker_evidence"], "pending") + self.assertNotIn("_embedding", segment) + + def test_a_b_a_keeps_three_time_ordered_blocks(self) -> None: + assembler = SegmentAssembler() + for sentence_id, speaker_id, start in ((1, 10, 0), (2, 20, 2000), (3, 10, 4000)): + assembler.apply_sentence( + { + "sentence_id": sentence_id, + "sentence": f"句子{sentence_id}", + "start_time": start, + "end_time": start + 1000, + "speaker_id": speaker_id, + "speaker_evidence": "confirmed", + "speaker_confidence": 0.9, + } + ) + + blocks = assembler.display_blocks() + self.assertEqual([block["speaker_id"] for block in blocks], [10, 20, 10]) + + +if __name__ == "__main__": + unittest.main() diff --git a/requirements-auxiliary.txt b/requirements-auxiliary.txt new file mode 100644 index 0000000..0354803 --- /dev/null +++ b/requirements-auxiliary.txt @@ -0,0 +1,6 @@ +aiohttp==3.11.11 +python-dotenv>=1.0 +funasr==1.3.1 +modelscope[framework]==1.34.0 +soundfile==0.13.1 +librosa==0.11.0 diff --git a/requirements-deploy.txt b/requirements-deploy.txt new file mode 100644 index 0000000..c6d1af1 --- /dev/null +++ b/requirements-deploy.txt @@ -0,0 +1,7 @@ +# 使用新版 VLLM 原生支持 Qwen3-ASR，避免 qwen-asr[vllm] 将 VLLM 锁定到 0.14.0。 +--extra-index-url https://download.pytorch.org/whl/cu130 +torch==2.13.0 +torchvision==0.28.0 +torchaudio==2.11.0 +vllm==0.28.0 +python-dotenv>=1.0 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..506e0cb --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""独立 Demo 项目的 VLLM 部署辅助模块。""" diff --git a/scripts/auxiliary_server.py b/scripts/auxiliary_server.py new file mode 100644 index 0000000..fe35446 --- /dev/null +++ b/scripts/auxiliary_server.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python3 +"""常驻加载 VAD、说话人聚类和声纹识别模型的独立服务。""" + +from __future__ import annotations + +import asyncio +import math +import os +import tempfile +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from aiohttp import web +from aiohttp.web_request import FileField +from dotenv import load_dotenv + +try: + from .model_manifest import auxiliary_models, load_manifest, model_directory +except ImportError: + from model_manifest import auxiliary_models, load_manifest, model_directory + + +# 将辅助服务端口固定在代码变量中，服务器启动时只需执行脚本，便于部署和排查。 +AUXILIARY_HOST = "0.0.0.0" +AUXILIARY_PORT = 8010 +PROJECT_ROOT = Path(__file__).resolve().parents[1] +# 独立启动辅助服务也必须读取部署配置，不能只在启动 vLLM 时才加载 .env。 +load_dotenv(PROJECT_ROOT / ".env") +MODELS_DIR = Path(os.getenv("MODEL_DIR", str(PROJECT_ROOT / "models"))).resolve() +AUXILIARY_DEVICE = os.getenv("AUXILIARY_DEVICE", "cuda:0") +ONLINE_SPEAKER_MATCH_THRESHOLD = 0.68 +MIN_ONLINE_SPEAKER_AUDIO_MS = 800 +# 实时 WebSocket 必须使用 VAD + CAM++ 声纹模型进行在线聚类。完整的 +# speech_campplus_speaker-diarization_common 是整段离线 diarization 接口， +# 与实时每个 turn 的 CAM++ embedding 不是同一加载路径；它仍可按需加载。 +DEFAULT_PRELOAD_KINDS = {"vad", "speaker_verification"} + + +def _coerce_finite_float(value: object) -> float | None: + """把表单或模型返回的数值安全转换为有限浮点数。""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + parsed = float(value) + elif isinstance(value, (str, bytes)): + try: + parsed = float(value.strip()) + except (TypeError, ValueError): + return None + else: + return None + return parsed if math.isfinite(parsed) else None + + +def _parse_form_float(value: object, field_name: str, default: float | None = None) -> float: + """解析 multipart 数值字段，避免直接把 FileField/bytes 传给 float。""" + parsed = _coerce_finite_float(value) + if parsed is not None: + return parsed + if default is not None: + return default + raise ValueError(f"{field_name} must be a number") + + +def _asset_ready(path: Path, config: dict[str, Any]) -> bool: + """在导入或加载模型前，先检查清单声明的文件和大小要求。""" + if not path.is_dir(): + return False + for relative_path in config.get("required_files", []): + if not (path / str(relative_path)).is_file(): + return False + any_files = config.get("any_files", []) + if any_files and not any( + file_path.is_file() + for pattern in any_files + for file_path in path.rglob(str(pattern)) + ): + return False + minimum_size = int(config.get("min_total_size_bytes", 0) or 0) + return not minimum_size or sum( + file_path.stat().st_size for file_path in path.rglob("*") if file_path.is_file() + ) >= minimum_size + + +class AuxiliaryRuntime: + """管理常驻辅助模型，并串行化 GPU 推理调用以避免显存竞争。""" + + def __init__(self) -> None: + self.manifest = load_manifest() + self.assets = auxiliary_models(self.manifest) + self.models: dict[str, Any] = {} + self.status: dict[str, dict[str, Any]] = {} + self.inference_lock = asyncio.Lock() + # 每个 WebSocket session 独立维护聚类中心，避免不同浏览器会话互相污染。 + self.speaker_clusters: dict[str, list[dict[str, Any]]] = {} + self.speaker_last_seen: dict[str, float] = {} + + def _preload_kinds(self) -> set[str]: + """读取需要在启动时加载的模型类型，默认不加载完整 diarization。""" + raw = os.getenv("AUXILIARY_PRELOAD_KINDS", "") + if not raw.strip(): + return set(DEFAULT_PRELOAD_KINDS) + # 无论环境变量如何设置，VAD 和 CAM++ speaker_verification 都是核心 + # 依赖；额外类型只会增加预加载项，不能绕过核心模型校验。 + return DEFAULT_PRELOAD_KINDS | {item.strip() for item in raw.split(",") if item.strip()} + + def _load_asset(self, model_id: str, config: dict[str, Any], path: Path) -> Any | None: + """只加载当前运行接口需要的模型；依赖模型和对齐模型先保持本地资产就绪。""" + kind = str(config.get("kind") or "") + if kind == "vad": + from funasr import AutoModel + + return AutoModel( + model=str(path), + device=AUXILIARY_DEVICE, + disable_update=True, + disable_pbar=True, + disable_log=True, + local_files_only=True, + ) + if kind in {"diarization", "speaker_verification", "realtime_speaker_verification"}: + from modelscope.pipelines import pipeline + from modelscope.utils.constant import Tasks + + task = Tasks.speaker_diarization if kind == "diarization" else Tasks.speaker_verification + return pipeline(task=task, model=str(path), device=AUXILIARY_DEVICE) + # CAM++ 依赖模型和 ForcedAligner 会先确认文件已落盘，后续由各自的专用 + # 推理路径使用；这里不猜测它们的通用加载方式，避免错误占用显存。 + return None + + def preload(self) -> None: + """预加载核心模型；可选模型失败只记录状态，避免服务整体退出。""" + failures: list[str] = [] + preload_kinds = self._preload_kinds() + loaded_kinds: set[str] = set() + for model_id, config in self.assets.items(): + path = model_directory(model_id, self.manifest, MODELS_DIR) + record: dict[str, Any] = {"path": str(path), "asset_ready": _asset_ready(path, config)} + kind = str(config.get("kind") or "") + if kind not in preload_kinds: + record["state"] = "optional_not_preloaded" if record["asset_ready"] else "optional_missing" + record["preload"] = False + self.status[model_id] = record + continue + # 清单中可能同时存在 iic/damo 两个同类型 CAM++ 资产；实时路径 + # 只需一份，按清单顺序选第一个成功加载的模型，避免重复占显存。 + if kind == "speaker_verification" and kind in loaded_kinds: + record["state"] = "duplicate_not_preloaded" + record["preload"] = False + self.status[model_id] = record + continue + record["preload"] = True + if not record["asset_ready"]: + record["state"] = "missing" + failures.append(model_id) + self.status[model_id] = record + continue + try: + loaded = self._load_asset(model_id, config, path) + if loaded is not None: + self.models[model_id] = loaded + record["state"] = "loaded" + loaded_kinds.add(kind) + else: + record["state"] = "load_error" + record["error"] = "model loader returned no model" + if kind in {"vad", "speaker_verification"}: + failures.append(model_id) + except Exception as exc: + record["state"] = "load_error" + record["error"] = str(exc) + failures.append(model_id) + self.status[model_id] = record + # 启动日志必须包含每个资产的路径、是否完整和底层异常；不能只打印 + # 一个笼统的“startup failed”，否则远程部署时无法判断缺文件还是版本错误。 + for model_id, record in self.status.items(): + print( + f"[model] {model_id}: state={record.get('state')}, " + f"asset_ready={record.get('asset_ready')}, path={record.get('path')}" + + (f", error={record['error']}" if record.get("error") else ""), + flush=True, + ) + # VAD 和 CAM++ speaker_verification 都是核心依赖，缺失/加载异常时 + # 立即失败并列出路径与底层错误，避免页面一直显示“未确认”。 + required_failures = [ + model_id for model_id in failures + if self.assets[model_id].get("kind") == "vad" + or ( + self.assets[model_id].get("kind") == "speaker_verification" + and self._speaker_embedding_model_id() is None + ) + ] + if required_failures: + details = "; ".join( + f"{model_id} -> {self.status[model_id]['path']}" + f" [{self.status[model_id].get('state')}: {self.status[model_id].get('error', 'asset missing')}]" + for model_id in required_failures + ) + raise RuntimeError( + "Auxiliary core model is missing or failed to load: " + details + + ". Run `python scripts/download_models.py --auxiliary-only` " + "or set MODEL_DIR to the directory containing the downloaded assets." + ) + + def _find_model(self, kind: str) -> Any: + """按模型清单中的 kind 查找一个已经加载完成的模型。""" + for model_id, config in self.assets.items(): + if config.get("kind") == kind and model_id in self.models: + return self.models[model_id] + raise RuntimeError(f"Auxiliary model is not loaded: {kind}") + + def _load_optional_kind(self, kind: str) -> Any: + """按需加载可选模型，例如完整 diarization 接口首次被调用时。""" + for model_id, config in self.assets.items(): + if config.get("kind") != kind: + continue + path = model_directory(model_id, self.manifest, MODELS_DIR) + if not _asset_ready(path, config): + raise RuntimeError(f"Auxiliary model asset is missing: {model_id} ({path})") + try: + loaded = self._load_asset(model_id, config, path) + except Exception as exc: + self.status.setdefault(model_id, {})["state"] = "load_error" + self.status[model_id]["error"] = str(exc) + raise RuntimeError(f"Auxiliary {kind} model failed to load: {exc}") from exc + if loaded is None: + raise RuntimeError(f"Auxiliary model has no loader for kind: {kind}") + self.models[model_id] = loaded + self.status.setdefault(model_id, {})["state"] = "loaded_on_demand" + return loaded + raise RuntimeError(f"Auxiliary model asset is not configured: {kind}") + + def _speaker_embedding_model_id(self) -> str | None: + """选择实时声纹模型，并在实时模型不可用时回退到普通声纹模型。""" + # CAM++ 是实时聚类的主模型；其它声纹模型不能静默替代它。 + for preferred_kind in ("speaker_verification",): + for model_id, config in self.assets.items(): + if config.get("kind") == preferred_kind and model_id in self.models: + return model_id + return None + + async def vad(self, audio_path: str) -> Any: + """使用临时音频文件执行一次串行化的 VAD 推理。""" + model = self._find_model("vad") + async with self.inference_lock: + return await asyncio.to_thread(model.generate, input=audio_path, cache={}) + + async def diarization(self, audio_path: str) -> Any: + """使用 CAM++ 对完整会话执行说话人聚类，保持跨片段的标签一致性。""" + try: + model = self._find_model("diarization") + except RuntimeError: + # 完整 diarization 不在核心启动路径，第一次调用接口时才加载。 + model = self._load_optional_kind("diarization") + async with self.inference_lock: + # ModelScope 的 CAM++ pipeline 以位置参数接收音频路径；使用关键字 + # input 在不同版本中可能被忽略或直接报参数错误。 + return await asyncio.to_thread(model, audio_path) + + @staticmethod + def _normalize_embedding(embedding: Any) -> Any: + """将声纹模型输出转成单位向量，并拒绝 NaN、无穷值和零向量。""" + import numpy as np + + vector = np.asarray(embedding, dtype=np.float32) + # 单次请求只能对应一条新声纹，不能把多个样本矩阵拼接成伪造特征。 + if vector.ndim > 2 or (vector.ndim == 2 and vector.shape[0] != 1): + raise RuntimeError("speaker embedding must contain exactly one vector") + vector = vector.reshape(-1) + if vector.size == 0 or not np.isfinite(vector).all(): + raise RuntimeError("speaker embedding is empty or non-finite") + norm = float(np.linalg.norm(vector)) + if not np.isfinite(norm) or norm < 1e-8: + raise RuntimeError("speaker embedding has zero norm") + return vector / norm + + @staticmethod + def _extract_embedding_value(result: Any) -> Any | None: + """从不同 ModelScope 版本的 pipeline 返回值中提取 embedding。""" + if result is None: + return None + + # ERes2Net pipeline 在 output_emb=True 时返回 {'embs': numpy.ndarray, + # 'outputs': ...}；部分版本或其它声纹 pipeline 使用 embedding 变体字段。 + if isinstance(result, Mapping): + for key in ("embs", "embedding", "embeddings", "speaker_embedding", "output_embedding"): + if key in result: + return AuxiliaryRuntime._extract_embedding_value(result[key]) + return None + + # 某些 ModelScope 版本把结果包装成带 embs/embedding 属性的对象。 + for key in ("embs", "embedding", "embeddings", "speaker_embedding", "output_embedding"): + value = getattr(result, key, None) + if value is not None: + return AuxiliaryRuntime._extract_embedding_value(value) + + # torch.Tensor 不能直接依赖 numpy.asarray 的 object 转换；先显式移到 CPU。 + detach = getattr(result, "detach", None) + if callable(detach): + detached = detach() + cpu = getattr(detached, "cpu", None) + if callable(cpu): + detached = cpu() + numpy_method = getattr(detached, "numpy", None) + if callable(numpy_method): + return numpy_method() + + # 单条音频通常返回 [embedding]，递归拆开这一层；数值列表则保留为向量。 + if isinstance(result, (list, tuple)) and len(result) == 1: + return AuxiliaryRuntime._extract_embedding_value(result[0]) + return result + + @staticmethod + def _run_embedding_pipeline(model_pipeline: Any, audio_path: str) -> Any: + """调用声纹 pipeline 的公开预处理和 embedding 输出接口。""" + # ModelScope 的 ERes2Net pipeline 要求输入为音频路径列表，并通过 + # output_emb=True 返回 embedding；不能直接把原始 waveform Tensor 喂给 + # pipeline.model，因为那会跳过采样率、声道和 waveform 预处理。 + try: + result = model_pipeline([audio_path], output_emb=True) + except TypeError: + # 兼容不支持 output_emb 参数的旧 pipeline：仍然使用 pipeline 自带 + # preprocess/forward，而不是直接调用内部 model，确保输入格式一致。 + preprocess = getattr(model_pipeline, "preprocess", None) + forward = getattr(model_pipeline, "forward", None) + if not callable(preprocess) or not callable(forward): + raise RuntimeError("speaker pipeline does not expose embedding inference") + result = forward(preprocess([audio_path])) + + embedding = AuxiliaryRuntime._extract_embedding_value(result) + if embedding is None: + raise RuntimeError( + "speaker pipeline returned no embedding " + f"(result_type={type(result).__name__})" + ) + return embedding + + def _extract_embedding_sync(self, audio_path: str) -> Any: + """在工作线程中读取当前 turn，并使用已加载的声纹模型提取特征。""" + import librosa + + audio, _ = librosa.load(audio_path, sr=16000, mono=True) + audio_array = audio.reshape(-1) + if audio_array.size < int(16000 * MIN_ONLINE_SPEAKER_AUDIO_MS / 1000): + return None + model_id = self._speaker_embedding_model_id() + if model_id is None: + raise RuntimeError("no loaded speaker verification model is available") + model_pipeline = self.models[model_id] + output = self._run_embedding_pipeline(model_pipeline, audio_path) + return self._normalize_embedding(output) + + async def resolve_speaker( + self, + audio_path: str, + session_id: str, + start_time_ms: float, + end_time_ms: float, + ) -> dict[str, Any] | None: + """对一个实时 turn 提取声纹，并更新该 session 的在线聚类中心。""" + async with self.inference_lock: + # 异常断网时客户端可能来不及 reset，过期状态在下一次请求时回收。 + now = time.monotonic() + for stale_id, seen in list(self.speaker_last_seen.items()): + if now - seen > 1800: + self.reset_speaker_session(stale_id) + self.speaker_last_seen[session_id] = now + embedding = await asyncio.to_thread(self._extract_embedding_sync, audio_path) + if embedding is None: + return {"speaker_id": -1, "speaker_evidence": "pending", "speaker_confidence": 0.0, + "speaker_status": "insufficient_audio", "speaker_reason": "音频不足 800ms，未提取声纹"} + import numpy as np + + # reset 可能在模型线程运行期间到达；结束的会话不能被晚到结果重新创建。 + if session_id not in self.speaker_last_seen: + return None + embedding = self._normalize_embedding(embedding) + clusters = self.speaker_clusters.setdefault(session_id, []) + best_cluster: dict[str, Any] | None = None + best_score = -1.0 + for cluster in clusters: + if embedding.shape != cluster["embedding"].shape: + raise RuntimeError("speaker embedding dimension changed within the session") + score = float(np.dot(embedding, cluster["embedding"])) + if score > best_score: + best_score = score + best_cluster = cluster + + if best_cluster is not None and best_score >= ONLINE_SPEAKER_MATCH_THRESHOLD: + count = int(best_cluster["count"]) + best_cluster["embedding"] = self._normalize_embedding( + (best_cluster["embedding"] * count) + embedding + ) + best_cluster["count"] = count + 1 + speaker_id = int(best_cluster["speaker_id"]) + confidence = best_score + strategy = "online_embedding_cluster_match" + else: + speaker_id = len(clusters) + clusters.append({"speaker_id": speaker_id, "embedding": embedding, "count": 1}) + confidence = 0.75 + strategy = "online_embedding_cluster_new" + + return { + "speaker_id": speaker_id, + "speaker_name": f"说话人 {speaker_id + 1}", + "speaker_evidence": "fresh", + "speaker_confidence": round(max(0.6, min(1.0, confidence)), 3), + "speaker_strategy": strategy, + "speaker_status": "confirmed", + "speaker_reason": "当前片段独立声纹已完成在线聚类", + "start_time": start_time_ms, + "end_time": end_time_ms, + } + + def reset_speaker_session(self, session_id: str) -> None: + """释放已结束 WebSocket 的聚类中心，防止长时间运行时内存增长。""" + self.speaker_clusters.pop(session_id, None) + self.speaker_last_seen.pop(session_id, None) + + +MODEL_SERVICE_KEY = web.AppKey("auxiliary_runtime", AuxiliaryRuntime) + + +async def health_handler(request: web.Request) -> web.Response: + """返回模型资产完整性和预加载状态，供 WebSocket 编排服务检查。""" + runtime: AuxiliaryRuntime = request.app[MODEL_SERVICE_KEY] + speaker_model_id = runtime._speaker_embedding_model_id() + vad_model_id = next( + (model_id for model_id, config in runtime.assets.items() + if config.get("kind") == "vad" and model_id in runtime.models), + None, + ) + # ready 表示实时链路的两个核心模型都可用；完整 diarization 是否 + # 预加载不影响这里的结果。 + ready = vad_model_id is not None and speaker_model_id is not None + return web.json_response( + { + "ready": ready, + "speaker_protocol_version": 2, + "vad_model": vad_model_id, + "vad_ready": vad_model_id is not None, + "device": AUXILIARY_DEVICE, + "speaker_embedding_model": speaker_model_id, + "speaker_embedding_ready": speaker_model_id is not None, + "models": runtime.status, + } + ) + + +async def vad_handler(request: web.Request) -> web.Response: + """接收 WAV 文件上传，并返回 FunASR 生成的语音活动区间。""" + runtime: AuxiliaryRuntime = request.app[MODEL_SERVICE_KEY] + form = await request.post() + upload = form.get("file") + if not isinstance(upload, FileField): + return web.json_response({"error": "multipart field 'file' is required"}, status=400) + temp_path: str | None = None + try: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + temp_file.write(upload.file.read()) + temp_path = temp_file.name + result = await runtime.vad(temp_path) + return web.json_response({"segments": result}) + finally: + if temp_path: + Path(temp_path).unlink(missing_ok=True) + + +def _normalize_diarization_segments(result: Any) -> list[dict[str, Any]]: + """将不同 ModelScope 版本的聚类输出统一为 start/end/speaker 字段。""" + # ModelScope 通常返回 {'text': [[start_sec, end_sec, speaker_id], ...]}； + # 某些版本返回对象而不是字典，因此这里同时读取属性形式。 + if isinstance(result, dict): + for key in ("segments", "output", "text", "result"): + candidate = result.get(key) + if isinstance(candidate, list): + result = candidate + break + else: + for key in ("segments", "output", "text", "result"): + candidate = getattr(result, key, None) + if isinstance(candidate, list): + result = candidate + break + if not isinstance(result, list): + return [] + + normalized: list[dict[str, Any]] = [] + for item in result: + values_are_milliseconds = False + if isinstance(item, dict): + values_are_milliseconds = "start_time" in item or "end_time" in item + start = item.get("start", item.get("start_time", item.get("begin"))) + end = item.get("end", item.get("end_time", item.get("stop"))) + speaker = item.get("speaker", item.get("speaker_id", item.get("label"))) + elif isinstance(item, (list, tuple)) and len(item) >= 3: + start, end, speaker = item[0], item[1], item[2] + else: + continue + start_value = _coerce_finite_float(start) + end_value = _coerce_finite_float(end) + if start_value is None or end_value is None: + continue + # 列表形式是 CAM++ 的秒单位；明确命名为 start_time/end_time 的 + # 字段按毫秒处理，避免用“超过多少数值”猜单位导致长录音误判。 + if values_are_milliseconds: + start_value /= 1000 + end_value /= 1000 + if end_value > start_value: + normalized.append({"start_time": round(start_value * 1000, 1), "end_time": round(end_value * 1000, 1), "speaker": str(speaker)}) + return normalized + + +async def diarization_handler(request: web.Request) -> web.Response: + """接收完整 WAV，返回 CAM++ 说话人聚类时间段。""" + runtime: AuxiliaryRuntime = request.app[MODEL_SERVICE_KEY] + form = await request.post() + upload = form.get("file") + if not isinstance(upload, FileField): + return web.json_response({"error": "multipart field 'file' is required"}, status=400) + temp_path: str | None = None + try: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + temp_file.write(upload.file.read()) + temp_path = temp_file.name + result = await runtime.diarization(temp_path) + return web.json_response({"segments": _normalize_diarization_segments(result)}) + finally: + if temp_path: + Path(temp_path).unlink(missing_ok=True) + + +async def speaker_resolve_handler(request: web.Request) -> web.Response: + """接收一个实时 turn，提取声纹并返回当前会话的在线聚类标签。""" + runtime: AuxiliaryRuntime = request.app[MODEL_SERVICE_KEY] + form = await request.post() + upload = form.get("file") + if not isinstance(upload, FileField): + return web.json_response({"error": "multipart field 'file' is required"}, status=400) + session_id = str(form.get("session_id") or "").strip() + if not session_id: + return web.json_response({"error": "multipart field 'session_id' is required"}, status=400) + try: + # aiohttp 的 MultiDictProxy 值可能是 str、bytes 或 FileField，先收窄 + # 为有限浮点数，避免静态检查告警和异常类型值进入声纹服务。 + start_time_ms = _parse_form_float(form.get("start_time_ms"), "start_time_ms", default=0.0) + end_time_ms = _parse_form_float(form.get("end_time_ms"), "end_time_ms", default=start_time_ms) + except ValueError: + return web.json_response({"error": "turn time fields must be numbers"}, status=400) + + temp_path: str | None = None + try: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: + temp_file.write(upload.file.read()) + temp_path = temp_file.name + result = await runtime.resolve_speaker( + temp_path, + session_id, + start_time_ms, + end_time_ms, + ) + return web.json_response(result or { + "speaker_id": -1, "speaker_evidence": "pending", "speaker_confidence": 0.0, + "speaker_status": "no_embedding", "speaker_reason": "当前片段未生成可用声纹", + }) + except Exception as exc: + # 将模型推理异常返回给 WebSocket 客户端，避免客户端只能看到笼统的 500。 + print( + f"[speaker] resolve failed: session_id={session_id}, " + f"start={start_time_ms}, end={end_time_ms}, error={exc}", + flush=True, + ) + return web.json_response({"error": str(exc)}, status=500) + finally: + if temp_path: + Path(temp_path).unlink(missing_ok=True) + + +async def speaker_reset_handler(request: web.Request) -> web.Response: + """释放已经结束的实时会话聚类中心。""" + runtime: AuxiliaryRuntime = request.app[MODEL_SERVICE_KEY] + payload = await request.json() + session_id = str(payload.get("session_id") or "").strip() if isinstance(payload, dict) else "" + if session_id: + runtime.reset_speaker_session(session_id) + return web.json_response({"ok": True}) + + +async def create_app() -> web.Application: + """创建辅助 HTTP 服务，并在服务启动前完成模型预加载。""" + runtime = AuxiliaryRuntime() + runtime.preload() + app = web.Application(client_max_size=64 * 1024 * 1024) + app[MODEL_SERVICE_KEY] = runtime + app.router.add_get("/health", health_handler) + app.router.add_post("/v1/vad", vad_handler) + app.router.add_post("/v1/diarization", diarization_handler) + app.router.add_post("/v1/speaker/resolve", speaker_resolve_handler) + app.router.add_post("/v1/speaker/reset", speaker_reset_handler) + return app + + +def main() -> None: + """启动宿主机上的常驻辅助模型服务。""" + print(f"Auxiliary model service: http://127.0.0.1:{AUXILIARY_PORT}", flush=True) + print(f"Device: {AUXILIARY_DEVICE}", flush=True) + web.run_app(create_app(), host=AUXILIARY_HOST, port=AUXILIARY_PORT) + + +if __name__ == "__main__": + main() diff --git a/scripts/download_models.py b/scripts/download_models.py new file mode 100644 index 0000000..e3159c1 --- /dev/null +++ b/scripts/download_models.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""为独立服务部署下载 ASR 和配套辅助模型。""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +# 同时支持直接执行脚本和 `python -m scripts.download_models` 两种方式， +# 下载器只依赖本目录中的清单模块，不耦合原项目的包路径。 +try: + from .model_manifest import auxiliary_models, load_manifest, model_directory, resolve_model_id +except ImportError: + from model_manifest import auxiliary_models, load_manifest, model_directory, resolve_model_id + + +def has_model_weights(model_path: Path) -> bool: + """检查 VLLM 加载 ASR 模型前必须存在的最小本地文件集合。""" + if not model_path.is_dir(): + return False + if not (model_path / "config.json").is_file(): + return False + return any(model_path.rglob("*.safetensors")) or any(model_path.rglob("*.bin")) + + +def is_model_ready(model_path: Path, config: dict[str, object]) -> bool: + """根据模型清单中的专属文件规则检查 ASR 或辅助资产是否完整。""" + if not model_path.is_dir(): + return False + required_files = config.get("required_files", []) + if isinstance(required_files, list): + for relative_path in required_files: + if not (model_path / str(relative_path)).is_file(): + return False + + any_files = config.get("any_files", []) + if isinstance(any_files, list) and any_files: + if not any( + file_path.is_file() + for pattern in any_files + for file_path in model_path.rglob(str(pattern)) + ): + return False + + minimum_size_value = config.get("min_total_size_bytes", 0) + # 模型清单使用 object 表示不同类型的资产字段，因此在转换为整数前必须 + # 先收窄类型，避免不合法的清单值在运行时触发难以定位的类型异常。 + minimum_size = ( + int(minimum_size_value) + if isinstance(minimum_size_value, (int, str)) + else 0 + ) + if minimum_size: + total_size = sum(file_path.stat().st_size for file_path in model_path.rglob("*") if file_path.is_file()) + if total_size < minimum_size: + return False + if required_files or any_files or minimum_size: + return True + return has_model_weights(model_path) + + +def download_model( + model_id: str, + model_path: Path, + cache_dir: Path | None, + revision: str | None, +) -> None: + """通过 ModelScope 下载一个指定资产，且不导入原项目应用代码。""" + # 延迟导入 ModelScope，使模型清单检查和单元测试无需安装重量级依赖。 + try: + from modelscope.hub.snapshot_download import snapshot_download + except ImportError as exc: + raise RuntimeError( + "ModelScope is required for downloading; install requirements-download.txt first" + ) from exc + + model_path.parent.mkdir(parents=True, exist_ok=True) + cache_path: str | None = None + if cache_dir is not None: + cache_dir.mkdir(parents=True, exist_ok=True) + cache_path = str(cache_dir) + print(f"Downloading model asset: {model_id}") + print(f"Local directory: {model_path}") + # 使用显式关键字参数而不是 **dict，既便于 Pylance 推断 ModelScope 的真实 + # 参数类型，也避免动态字典被误判为其它无关参数的类型签名。 + snapshot_download( + model_id, + revision=revision, + cache_dir=cache_path, + local_dir=str(model_path), + ) + + +def fix_camplusplus_config(models_dir: Path) -> bool: + """将 CAM++ 依赖模型 ID 改写为本地路径，确保服务可以离线启动。 + + 聚类流水线会在 ``configuration.json`` 中保存多个 ModelScope 模型 ID。 + 如果不改写这些 ID，即使所有文件已经下载完整，辅助服务在无网络环境 + 启动时仍可能再次访问 ModelScope 获取依赖。 + """ + config_file = models_dir / "iic/speech_campplus_speaker-diarization_common/configuration.json" + if not config_file.is_file(): + return False + + replacements = { + "damo/speech_campplus_sv_zh-cn_16k-common": models_dir / "damo/speech_campplus_sv_zh-cn_16k-common", + "iic/speech_campplus_sv_zh-cn_16k-common": models_dir / "iic/speech_campplus_sv_zh-cn_16k-common", + "damo/speech_campplus-transformer_scl_zh-cn_16k-common": models_dir / "damo/speech_campplus-transformer_scl_zh-cn_16k-common", + "damo/speech_campplus-transformer_scl_zh-cn-16k-common": models_dir / "damo/speech_campplus-transformer_scl_zh-cn-16k-common", + "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch": models_dir / "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", + } + try: + config = json.loads(config_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"Unable to read CAM++ configuration: {exc}") + return False + + raw_model_config = config.get("model") + if not isinstance(raw_model_config, dict): + return False + model_config: dict[str, object] = { + str(key): value for key, value in raw_model_config.items() + } + modified = False + for key in ("speaker_model", "change_locator", "vad_model"): + old_value = model_config.get(key) + local_path = replacements.get(old_value) if isinstance(old_value, str) else None + if local_path is not None and local_path.exists(): + model_config[key] = str(local_path) + modified = True + if not modified: + return False + config["model"] = model_config + config_file.write_text(json.dumps(config, indent=4, ensure_ascii=False) + "\n", encoding="utf-8") + return True + + +def main() -> int: + """检查或下载 ASR 模型及辅助运行时所需的全部资产。""" + # 保持当前部署项目与原项目模型规划器完全独立，同时将孤立服务需要的 + # 模型统一准备到本地，方便后续在服务器上离线启动多个常驻服务。 + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + default=os.getenv("QWEN3_ASR_MODEL", "default"), + help="ASR model alias (1.7b/0.6b), exact model ID, or default", + ) + parser.add_argument( + "--models-dir", + type=Path, + default=Path(os.getenv("MODEL_DIR", str(Path(__file__).resolve().parents[1] / "models"))), + help="Root directory for local model files", + ) + model_scope_cache = os.getenv("MODELSCOPE_CACHE") + parser.add_argument( + "--cache-dir", + type=Path, + default=Path(model_scope_cache) if model_scope_cache else None, + help="Optional ModelScope cache directory", + ) + parser.add_argument( + "--check-only", + action="store_true", + help="Only check selected assets; do not download", + ) + auxiliary_group = parser.add_mutually_exclusive_group() + auxiliary_group.add_argument( + "--skip-auxiliary", + action="store_true", + help="Only download/check the selected ASR model", + ) + auxiliary_group.add_argument( + "--auxiliary-only", + action="store_true", + help="Only download/check VAD, speaker, diarization, and aligner assets", + ) + args = parser.parse_args() + + manifest = load_manifest() + models_dir = args.models_dir.resolve() + cache_dir = args.cache_dir.resolve() if args.cache_dir else None + selected_assets: list[tuple[str, dict[str, object]]] = [] + if not args.auxiliary_only: + model_id = resolve_model_id(args.model, manifest) + selected_assets.append((model_id, manifest["models"][model_id])) + if not args.skip_auxiliary: + selected_assets.extend(auxiliary_models(manifest).items()) + + missing: list[tuple[str, Path, dict[str, object]]] = [] + for model_id, config in selected_assets: + model_path = model_directory(model_id, manifest, models_dir) + if is_model_ready(model_path, config): + print(f"Model asset is ready: {model_id}") + else: + missing.append((model_id, model_path, config)) + + if not missing: + # 即使资产已经存在，也要重新执行一次离线配置修正；这样从其它主机 + # 复制过来的模型包也能在启动辅助服务前自动完成本地路径修复。 + if fix_camplusplus_config(models_dir): + print("CAM++ configuration updated for offline local model paths") + print(f"All selected model assets are ready: {len(selected_assets)}") + return 0 + if args.check_only: + for model_id, model_path, _ in missing: + print(f"Model asset is missing or incomplete: {model_id} ({model_path})") + return 1 + + failed: list[str] = [] + for model_id, model_path, config in missing: + try: + revision = str(config.get("revision") or "") or None + download_model(model_id, model_path, cache_dir, revision) + if not is_model_ready(model_path, config): + print(f"Download finished but model asset is incomplete: {model_path}") + failed.append(model_id) + else: + print(f"Model asset is ready: {model_id}") + except Exception as exc: + print(f"Download failed: {model_id}: {exc}") + failed.append(model_id) + if not failed and fix_camplusplus_config(models_dir): + print("CAM++ configuration updated for offline local model paths") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/model_manifest.py b/scripts/model_manifest.py new file mode 100644 index 0000000..d4e7d18 --- /dev/null +++ b/scripts/model_manifest.py @@ -0,0 +1,54 @@ +"""独立 VLLM 部署项目的模型清单与路径解析辅助函数。""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MANIFEST_PATH = PROJECT_ROOT / "model_manifest.json" + + +def load_manifest(path: Path = MANIFEST_PATH) -> dict[str, Any]: + """读取本项目自己的模型清单，整个过程不导入原项目代码。""" + with path.open("r", encoding="utf-8") as manifest_file: + manifest = json.load(manifest_file) + if not isinstance(manifest.get("models"), dict) or not manifest["models"]: + raise ValueError("model_manifest.json 必须包含非空的 models 对象") + return manifest + + +def resolve_model_id(model: str | None, manifest: dict[str, Any]) -> str: + """将默认值、短别名或完整模型 ID 解析为一个 ASR 模型。""" + models = manifest["models"] + requested = (model or "default").strip() + if requested == "default": + requested = str(manifest["default_model"]) + + if requested in models: + return requested + + for model_id, config in models.items(): + if requested.lower() == str(config.get("alias", "")).lower(): + return model_id + raise ValueError(f"不支持的 ASR 模型 '{model}'，可选模型：{', '.join(models)}") + + +def model_directory(model_id: str, manifest: dict[str, Any], models_dir: Path) -> Path: + """根据清单返回 ASR 或辅助模型实际使用的本地目录。""" + config = manifest.get("models", {}).get(model_id) + if config is None: + config = manifest.get("auxiliary_models", {}).get(model_id) + if not isinstance(config, dict) or not config.get("directory"): + raise ValueError(f"模型 '{model_id}' 在清单中没有配置本地目录") + return models_dir / str(config["directory"]) + + +def auxiliary_models(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]: + """返回可独立部署的 VAD、说话人和对齐模型资产。""" + models = manifest.get("auxiliary_models", {}) + if not isinstance(models, dict): + raise ValueError("model_manifest.json 的 auxiliary_models 必须是对象") + return {str(model_id): config for model_id, config in models.items() if isinstance(config, dict)} diff --git a/scripts/serve.py b/scripts/serve.py new file mode 100644 index 0000000..ffbf6f6 --- /dev/null +++ b/scripts/serve.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""在宿主机启动独立的 Qwen3-ASR VLLM 服务。""" + +from __future__ import annotations + +import argparse +import os +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path +from urllib.error import URLError +from urllib.request import urlopen + +from dotenv import load_dotenv + + +# 启动器自动读取 demo/.env；系统环境变量仍然优先，便于部署平台临时覆盖配置。 +PROJECT_ROOT = Path(__file__).resolve().parents[1] +load_dotenv(PROJECT_ROOT / ".env") + +# 将服务端口集中在代码变量中维护，启动时不需要额外传入端口参数；健康检查、 +# VLLM 子进程命令和就绪提示都使用同一个端口，避免配置不一致导致误判。 +SERVER_PORT = int(os.getenv("VLLM_PORT", "9950")) + +try: + from .model_manifest import load_manifest, model_directory, resolve_model_id +except ImportError: + from model_manifest import load_manifest, model_directory, resolve_model_id + + +def has_model_weights(model_path: Path) -> bool: + """检查 VLLM 加载模型前必须存在的最小本地文件集合。""" + if not model_path.is_dir() or not (model_path / "config.json").is_file(): + return False + return any(model_path.rglob("*.safetensors")) or any(model_path.rglob("*.bin")) + + +def positive_int(value: str) -> int: + """解析启动轮询使用的正整数参数，并拒绝零和负数。""" + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least 1") + return parsed + + +def non_negative_float(value: str) -> float: + """解析启动轮询间隔，并拒绝会导致逻辑异常的负数。""" + parsed = float(value) + if parsed < 0: + raise argparse.ArgumentTypeError("value must be non-negative") + return parsed + + +def build_parser() -> argparse.ArgumentParser: + """创建宿主机启动参数解析器，默认值允许通过环境变量统一覆盖。""" + parser = argparse.ArgumentParser(description=__doc__) + add_arguments(parser) + return parser + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + """注册模型、网络端点和启动检查循环相关的命令行参数。""" + parser.add_argument( + "--model", + default=os.getenv("QWEN3_ASR_MODEL", "default"), + help="Model alias, exact model ID, or default", + ) + parser.add_argument( + "--models-dir", + type=Path, + default=Path(os.getenv("MODEL_DIR", str(Path(__file__).resolve().parents[1] / "models"))), + help="Root directory containing downloaded model files", + ) + parser.add_argument("--host", default=os.getenv("VLLM_HOST", "0.0.0.0")) + parser.add_argument( + "--display-host", + default=os.getenv("VLLM_DISPLAY_HOST", "127.0.0.1"), + help="Host name shown in the ready message; does not change the bind address", + ) + parser.add_argument( + "--probe-host", + default=os.getenv("VLLM_PROBE_HOST", "127.0.0.1"), + help="Host used by the startup health probe", + ) + parser.add_argument( + "--startup-check-loops", + type=positive_int, + default=positive_int(os.getenv("VLLM_STARTUP_CHECK_LOOPS", "60")), + help="Maximum number of health checks before startup fails", + ) + parser.add_argument( + "--startup-check-interval", + type=non_negative_float, + default=non_negative_float(os.getenv("VLLM_STARTUP_CHECK_INTERVAL_SECONDS", "2")), + help="Seconds between startup health checks", + ) + parser.add_argument("--served-model-name", default=os.getenv("VLLM_SERVED_MODEL_NAME")) + parser.add_argument( + "--gpu-memory-utilization", + default=os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.3"), + ) + parser.add_argument("--max-model-len", default=os.getenv("VLLM_MAX_MODEL_LEN", "16384")) + parser.add_argument("--max-num-seqs", default=os.getenv("VLLM_MAX_NUM_SEQS", "16")) + parser.add_argument("--tensor-parallel-size", default=os.getenv("VLLM_TENSOR_PARALLEL_SIZE", "1")) + parser.add_argument( + "--enforce-eager", + action=argparse.BooleanOptionalAction, + default=os.getenv("VLLM_ENFORCE_EAGER", "true").lower() == "true", + ) + + +def build_server_command(args: argparse.Namespace, model_id: str, model_path: Path) -> list[str]: + """构造新版 VLLM 原生启动命令，不依赖 qwen-asr-serve。""" + # Qwen3-ASR 已由新版 VLLM 原生支持，因此这里调用 vllm serve，避免 + # qwen-asr-serve 对旧版 VLLM 的固定依赖影响 GB10 部署环境。 + executable_name = os.getenv("VLLM_EXECUTABLE", "vllm") + executable = shutil.which(executable_name) + if executable is None: + raise RuntimeError(f"{executable_name} was not found; install requirements-deploy.txt first") + + served_model_name = args.served_model_name or model_id + command = [ + executable, + "serve", + str(model_path), + "--host", + args.host, + "--port", + str(SERVER_PORT), + "--served-model-name", + served_model_name, + "--gpu-memory-utilization", + str(args.gpu_memory_utilization), + "--max-model-len", + str(args.max_model_len), + "--max-num-seqs", + str(args.max_num_seqs), + "--tensor-parallel-size", + str(args.tensor_parallel_size), + ] + if args.enforce_eager: + command.append("--enforce-eager") + return command + + +def wait_until_ready(process: subprocess.Popen[bytes], probe_url: str, loops: int, interval: float) -> None: + """按调用方指定的次数和间隔轮询 VLLM 健康接口，直到服务就绪或失败。""" + for attempt in range(1, loops + 1): + if process.poll() is not None: + raise RuntimeError(f"VLLM exited during startup with code {process.returncode}") + try: + with urlopen(probe_url, timeout=2) as response: + if 200 <= response.status < 300: + return + except (OSError, URLError): + pass + print(f"Waiting for VLLM startup ({attempt}/{loops})...", flush=True) + if attempt < loops: + time.sleep(interval) + raise TimeoutError(f"VLLM did not become ready after {loops} health checks: {probe_url}") + + +def stop_process(process: subprocess.Popen[bytes]) -> None: + """向 VLLM 子进程转发优雅停止信号，并在超时后执行兜底清理。""" + if process.poll() is not None: + return + if os.name == "nt": + process.send_signal(signal.CTRL_BREAK_EVENT) + else: + process.send_signal(signal.SIGINT) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=10) + + +def main() -> int: + """解析一个本地模型、启动 VLLM，并持续托管宿主机子进程。""" + args = build_parser().parse_args() + + manifest = load_manifest() + model_id = resolve_model_id(args.model, manifest) + model_path = model_directory(model_id, manifest, args.models_dir.resolve()) + if not has_model_weights(model_path): + print(f"Model is missing or incomplete: {model_id} ({model_path})", file=sys.stderr) + print("Run scripts/download_models.py for the same model first.", file=sys.stderr) + return 1 + + command = build_server_command(args, model_id, model_path) + probe_url = f"http://{args.probe_host}:{SERVER_PORT}/health" + display_url = f"http://{args.display_host}:{SERVER_PORT}" + print(f"Starting Qwen3-ASR VLLM service on host: {display_url}", flush=True) + print(f"Model: {model_id}", flush=True) + print(f"Startup checks: {args.startup_check_loops} x {args.startup_check_interval}s", flush=True) + + process = subprocess.Popen(command) + try: + wait_until_ready(process, probe_url, args.startup_check_loops, args.startup_check_interval) + print(f"VLLM ready: {display_url}", flush=True) + print(f"OpenAI endpoint: {display_url}/v1", flush=True) + while process.poll() is None: + time.sleep(0.5) + return int(process.returncode or 0) + except (KeyboardInterrupt, TimeoutError, RuntimeError) as exc: + print(str(exc), file=sys.stderr) + return 1 + finally: + stop_process(process) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_auxiliary_server.py b/tests/test_auxiliary_server.py new file mode 100644 index 0000000..08e4ed0 --- /dev/null +++ b/tests/test_auxiliary_server.py @@ -0,0 +1,169 @@ +"""辅助服务输出格式测试，确保不同 ModelScope 版本都能被统一解析。""" + +from __future__ import annotations + +import unittest +import numpy as np +from types import SimpleNamespace +from unittest.mock import patch + +from scripts.auxiliary_server import ( + AuxiliaryRuntime, + _coerce_finite_float, + _normalize_diarization_segments, + _parse_form_float, +) + + +class AuxiliaryServerTests(unittest.TestCase): + """验证 CAM++ 常见的秒、毫秒和对象返回格式。""" + + def test_multipart_numeric_values_are_narrowed_before_model_calls(self): + """表单值只接受有限数字，FileField 或非法文本回退/报错而不传入 float。""" + self.assertEqual(_coerce_finite_float(" 12.5 "), 12.5) + self.assertEqual(_coerce_finite_float(b"12.5"), 12.5) + self.assertIsNone(_coerce_finite_float(float("nan"))) + self.assertEqual(_parse_form_float(None, "start_time_ms", default=0.0), 0.0) + with self.assertRaises(ValueError): + _parse_form_float("not-a-number", "start_time_ms") + + def test_rejects_invalid_embedding_vectors(self): + """模型成功返回也不代表向量有效，异常特征不能污染聚类池。""" + for value in ([0, 0], [], [float("nan"), 1], [float("inf"), 0], [[1, 0], [0, 1]]): + with self.assertRaises(RuntimeError): + AuxiliaryRuntime._normalize_embedding(value) + + def test_optional_diarization_failure_does_not_block_core_startup(self): + """完整 CAM++ diarization 不是实时核心加载路径，启动失败应只记录可选状态。""" + runtime = AuxiliaryRuntime() + runtime.assets = { + "vad": {"kind": "vad"}, + "diarization": {"kind": "diarization"}, + "aligner": {"kind": "forced_aligner"}, + } + loaded_kinds = [] + with patch("scripts.auxiliary_server._asset_ready", return_value=True), \ + patch("scripts.auxiliary_server.model_directory", return_value=runtime.manifest and SimpleNamespace()), \ + patch.object(runtime, "_load_asset", side_effect=lambda model_id, config, path: loaded_kinds.append(config["kind"]) or object()): + runtime.preload() + self.assertEqual(loaded_kinds, ["vad"]) + self.assertEqual(runtime.status["diarization"]["state"], "optional_not_preloaded") + self.assertEqual(runtime.status["aligner"]["state"], "optional_not_preloaded") + + def test_campplus_speaker_is_a_required_core_model(self): + """实时在线聚类必须使用 CAM++ speaker_verification，不能只启动 VAD。""" + runtime = AuxiliaryRuntime() + runtime.assets = { + "vad": {"kind": "vad"}, + "campplus": {"kind": "speaker_verification"}, + } + with patch("scripts.auxiliary_server._asset_ready", side_effect=[True, False]), \ + patch("scripts.auxiliary_server.model_directory", return_value=SimpleNamespace()): + with self.assertRaisesRegex(RuntimeError, "campplus"): + runtime.preload() + + def test_campplus_is_preferred_over_other_embedding_models(self): + """即使同时存在 ERes2Net，实时聚类仍优先使用 CAM++。""" + runtime = AuxiliaryRuntime() + runtime.assets = { + "campplus": {"kind": "speaker_verification"}, + "eres2net": {"kind": "realtime_speaker_verification"}, + } + runtime.models = {"campplus": object(), "eres2net": object()} + self.assertEqual(runtime._speaker_embedding_model_id(), "campplus") + + def test_missing_vad_reports_download_path(self): + """VAD 是核心依赖，缺失时错误必须给出可执行的修复方向。""" + runtime = AuxiliaryRuntime() + runtime.assets = {"vad": {"kind": "vad"}} + with patch("scripts.auxiliary_server._asset_ready", return_value=False), \ + patch("scripts.auxiliary_server.model_directory", return_value=SimpleNamespace(__str__=lambda self: "/models/vad")): + with self.assertRaisesRegex(RuntimeError, "download_models.py --auxiliary-only"): + runtime.preload() + + def test_uses_public_pipeline_embedding_output(self) -> None: + """声纹推理必须走 pipeline 的预处理和 output_emb 接口。""" + + class FakePipeline: + def __init__(self) -> None: + self.calls: list[tuple[list[str], bool]] = [] + + def __call__(self, audio_paths: list[str], output_emb: bool = False) -> dict[str, object]: + self.calls.append((audio_paths, output_emb)) + return {"outputs": {"text": "No similarity score output"}, "embs": [[1.0, 2.0, 3.0]]} + + pipeline = FakePipeline() + result = AuxiliaryRuntime._run_embedding_pipeline(pipeline, "turn.wav") + + self.assertEqual(result, [1.0, 2.0, 3.0]) + self.assertEqual(pipeline.calls, [(["turn.wav"], True)]) + + def test_supports_legacy_pipeline_without_output_emb_argument(self) -> None: + """旧版 pipeline 不支持 output_emb 时，仍应使用其 preprocess/forward。""" + + class LegacyPipeline: + def __init__(self) -> None: + self.prepared: list[str] = [] + + def __call__(self, *_args: object, **_kwargs: object) -> None: + raise TypeError("output_emb is not supported") + + def preprocess(self, audio_paths: list[str]) -> list[str]: + return [f"prepared:{audio_paths[0]}"] + + def forward(self, prepared: list[str]) -> list[list[float]]: + self.prepared = prepared + return [[0.1, 0.2, 0.3]] + + pipeline = LegacyPipeline() + result = AuxiliaryRuntime._run_embedding_pipeline(pipeline, "turn.wav") + + self.assertEqual(result, [0.1, 0.2, 0.3]) + self.assertEqual(pipeline.prepared, ["prepared:turn.wav"]) + + def test_normalizes_modelscope_text_seconds(self) -> None: + result = _normalize_diarization_segments({"text": [[0.2, 1.4, 0], [1.4, 2.0, 1]]}) + self.assertEqual( + result, + [ + {"start_time": 200.0, "end_time": 1400.0, "speaker": "0"}, + {"start_time": 1400.0, "end_time": 2000.0, "speaker": "1"}, + ], + ) + + def test_normalizes_named_millisecond_fields(self) -> None: + result = _normalize_diarization_segments( + {"segments": [{"start_time": 100, "end_time": 900, "speaker_id": "cluster-a"}]} + ) + self.assertEqual(result[0]["start_time"], 100.0) + self.assertEqual(result[0]["end_time"], 900.0) + + def test_reads_object_text_attribute(self) -> None: + result = _normalize_diarization_segments(SimpleNamespace(text=[[1, 2, "spk"]])) + self.assertEqual(result[0]["speaker"], "spk") + + +class OnlineSpeakerTests(unittest.IsolatedAsyncioTestCase): + """使用独立的新鲜向量验证 A→B→A，无需显卡和模型权重。""" + + async def test_fresh_embeddings_preserve_a_b_a_and_reset(self): + runtime = AuxiliaryRuntime() + vectors = iter(([1, 0], [0, 1], [0.99, 0.01])) + runtime._extract_embedding_sync = lambda _: np.array(next(vectors), dtype=np.float32) + results = [await runtime.resolve_speaker("turn.wav", "test", i * 2000, i * 2000 + 1000) for i in range(3)] + self.assertEqual([r["speaker_id"] for r in results], [0, 1, 0]) + self.assertEqual([c["count"] for c in runtime.speaker_clusters["test"]], [2, 1]) + runtime.reset_speaker_session("test") + self.assertNotIn("test", runtime.speaker_clusters) + + async def test_missing_embedding_does_not_copy_previous_cluster(self): + runtime = AuxiliaryRuntime() + runtime._extract_embedding_sync = lambda _: np.array([1, 0], dtype=np.float32) + await runtime.resolve_speaker("turn.wav", "test", 0, 1000) + runtime._extract_embedding_sync = lambda _: None + result = await runtime.resolve_speaker("short.wav", "test", 1000, 1200) + self.assertEqual(result["speaker_id"], -1) + self.assertEqual(runtime.speaker_clusters["test"][0]["count"], 1) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_model_manifest.py b/tests/test_model_manifest.py new file mode 100644 index 0000000..b8a404b --- /dev/null +++ b/tests/test_model_manifest.py @@ -0,0 +1,73 @@ +"""独立模型清单测试，确保不会导入原项目应用。""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.download_models import fix_camplusplus_config +from scripts.model_manifest import auxiliary_models, load_manifest, model_directory, resolve_model_id + + +class ModelManifestTests(unittest.TestCase): + def setUp(self) -> None: + self.manifest = load_manifest() + + def test_default_is_zero_point_six_b_model(self) -> None: + self.assertEqual(resolve_model_id("default", self.manifest), "Qwen/Qwen3-ASR-0.6B") + + def test_aliases_resolve_to_individual_models(self) -> None: + self.assertEqual(resolve_model_id("1.7b", self.manifest), "Qwen/Qwen3-ASR-1.7B") + self.assertEqual(resolve_model_id("0.6b", self.manifest), "Qwen/Qwen3-ASR-0.6B") + + def test_manifest_has_only_asr_models(self) -> None: + self.assertEqual( + set(self.manifest["models"]), + {"Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-1.7B"}, + ) + + def test_manifest_has_auxiliary_runtime_assets(self) -> None: + assets = auxiliary_models(self.manifest) + self.assertIn("damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", assets) + self.assertIn("iic/speech_campplus_speaker-diarization_common", assets) + self.assertIn("iic/speech_campplus_sv_zh-cn_16k-common", assets) + self.assertIn("Qwen/Qwen3-ForcedAligner-0.6B", assets) + + def test_model_directory_is_under_demo_models(self) -> None: + models_dir = Path(__file__).resolve().parents[1] / "models" + for model_id in [*self.manifest["models"], *auxiliary_models(self.manifest)]: + self.assertTrue(model_directory(model_id, self.manifest, models_dir).is_relative_to(models_dir)) + + def test_camplusplus_config_is_rewritten_to_local_assets(self) -> None: + """离线模型包不能继续从 ModelScope 解析 CAM++ 依赖。""" + with tempfile.TemporaryDirectory() as temp_dir: + models_dir = Path(temp_dir) + config_dir = models_dir / "iic/speech_campplus_speaker-diarization_common" + config_dir.mkdir(parents=True) + for relative_path in ( + "damo/speech_campplus_sv_zh-cn_16k-common", + "iic/speech_campplus_sv_zh-cn_16k-common", + "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", + ): + (models_dir / relative_path).mkdir(parents=True) + config = { + "model": { + "speaker_model": "iic/speech_campplus_sv_zh-cn_16k-common", + "change_locator": "damo/speech_campplus_sv_zh-cn_16k-common", + "vad_model": "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", + } + } + (config_dir / "configuration.json").write_text(json.dumps(config), encoding="utf-8") + + self.assertTrue(fix_camplusplus_config(models_dir)) + updated = json.loads((config_dir / "configuration.json").read_text(encoding="utf-8")) + self.assertEqual( + updated["model"]["speaker_model"], + str(models_dir / "iic/speech_campplus_sv_zh-cn_16k-common"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_serve.py b/tests/test_serve.py new file mode 100644 index 0000000..dbed560 --- /dev/null +++ b/tests/test_serve.py @@ -0,0 +1,43 @@ +"""宿主机 VLLM 启动配置测试。""" + +from __future__ import annotations + +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +from scripts.serve import SERVER_PORT, build_parser, build_server_command + + +class ServeConfigTests(unittest.TestCase): + def test_host_and_startup_loop_are_read_from_environment(self) -> None: + values = { + "VLLM_HOST": "192.168.1.10", + "VLLM_DISPLAY_HOST": "asr.local", + "VLLM_STARTUP_CHECK_LOOPS": "12", + "VLLM_STARTUP_CHECK_INTERVAL_SECONDS": "0.5", + } + with patch.dict(os.environ, values, clear=False): + parser = build_parser() + args = parser.parse_args([]) + + self.assertEqual(args.host, "192.168.1.10") + self.assertEqual(SERVER_PORT, 9950) + self.assertNotIn("--port", parser.format_help()) + self.assertEqual(args.display_host, "asr.local") + self.assertEqual(args.startup_check_loops, 12) + self.assertEqual(args.startup_check_interval, 0.5) + + @patch("scripts.serve.shutil.which", return_value="/opt/asr-gb10/bin/vllm") + def test_builds_native_vllm_serve_command(self, _which: object) -> None: + """启动器应生成已验证的新版 vllm serve 命令。""" + args = build_parser().parse_args([]) + command = build_server_command(args, "Qwen/Qwen3-ASR-0.6B", Path("/models/Qwen3-ASR-0.6B")) + + self.assertEqual(command[0:3], ["/opt/asr-gb10/bin/vllm", "serve", str(Path("/models/Qwen3-ASR-0.6B"))]) + self.assertIn("--enforce-eager", command) + self.assertIn("9950", command) + +if __name__ == "__main__": + unittest.main()