From 9614b2d2253387153a82db93ae227732c584cd15 Mon Sep 17 00:00:00 2001 From: "mula.liu" Date: Thu, 5 Nov 2026 16:53:24 +0800 Subject: [PATCH] v 0.1.1 --- .env.example | 5 +- README.md | 24 +- backend/app/api/library.py | 3 +- backend/app/api/platform.py | 6 +- backend/app/api/system.py | 4 +- backend/app/jobs/handlers.py | 8 +- backend/app/providers/base.py | 3 + backend/app/providers/comfy_assets.py | 66 ++- backend/app/providers/macos_say.py | 119 ----- backend/app/providers/registry.py | 55 +- backend/app/services/audio.py | 5 +- backend/app/services/dispatch.py | 2 +- backend/app/services/timbre.py | 51 +- backend/tests/test_index_tts_instruct.py | 190 +++++++ backend/tests/test_macos_say.py | 77 --- docs/UI规范.md | 65 ++- docs/工程规范.md | 86 +++- frontend/src/components/media/WorkflowBar.tsx | 50 +- frontend/src/components/ui/Status.tsx | 4 +- frontend/src/components/ui/Steps.tsx | 61 ++- frontend/src/pages/AssetLibrary.tsx | 192 +++---- frontend/src/pages/ModelCenter.tsx | 276 +++++----- frontend/src/pages/project/Library.tsx | 10 +- frontend/src/pages/project/Storyboard.tsx | 9 +- frontend/src/stores/models.ts | 5 +- scripts/deploy.sh | 137 ++++- scripts/seed.py | 476 +++++++++++++++++- scripts/server.sh | 34 +- scripts/ui/btnclustercheck.py | 139 +++++ scripts/ui/gateprobe.py | 207 +++++++- scripts/ui/gridcheck.py | 82 ++- scripts/ui/helpshots.py | 18 +- scripts/ui/tableaudit.py | 32 ++ scripts/ui/tagcheck.py | 79 ++- scripts/ui/visualcheck.py | 7 + 35 files changed, 1982 insertions(+), 605 deletions(-) delete mode 100644 backend/app/providers/macos_say.py create mode 100644 backend/tests/test_index_tts_instruct.py delete mode 100644 backend/tests/test_macos_say.py create mode 100644 scripts/ui/btnclustercheck.py diff --git a/.env.example b/.env.example index 6585d98..b68c8c8 100644 --- a/.env.example +++ b/.env.example @@ -75,8 +75,9 @@ COMFYUI_RPM=6 # VIDEO_MODEL=minimax-h3 # ---------- 配音 TTS ---------- -# macOS 本机真实人声可设为 macos-say;声线在项目的「资产库 → 声线」中配置 -# (Tingting / Meijia / Sinji / Fred,params.effect=alien 可生成外星语效果)。 +# 真实配音用 comfyui_tts(IndexTTS2,部署机 ComfyUI 提供,Linux 可用); +# 声线在「资产库 → 声线」配置:绑真人参考音频走 zero-shot 克隆, +# 只填「音色描述」则由描述词直出。 TTS_PROVIDER=mock TTS_API_BASE_URL= TTS_API_KEY= diff --git a/README.md b/README.md index 14034fb..b91324d 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ DEV-only,拿 `NEX_UI_BASE=:4173` 量构建产物时它恒为 0,别把那份 | `scripts/dev.sh` | 开发态前后端两个进程(后端 8000 + Vite 5173),缺 `.env` / `.venv` / `node_modules` 自己补 | 开发机 | | `scripts/server.sh` | 服务形态全生命周期:`install` `build` `start` `stop` `restart` `status` `verify` `logs` `systemd` | 服务器 | | `scripts/deploy.sh` | 打部署包(白名单 + 体检),或直接 rsync 上服务器并自动安装、重启 | 开发机 | -| `scripts/seed.py` | 迁移「模型配置 + 加密主密钥 + 管理员」:`export` 导出、`install` 幂等落地 | 两端 | +| `scripts/seed.py` | 迁移「模型配置 + 加密主密钥 + 管理员」:`doctor` 体检数据集与代码是否对得上、`export --fix` 校准并导出、`install` 装前自动体检再幂等落地、`boundary` 数据边界一览 | 两端 | | `scripts/e2e_selfcheck.py` | 图2 全 12 节点端到端自检(真实 HTTP + ffprobe 实测产物) | 两端 | | `scripts/evidence_prune.py` | 清理 `reports/` 下过期的验收证据目录 | 开发机 | @@ -328,15 +328,17 @@ React Router 的 history 回退已经处理好(地址栏直接刷 `/projects/3 `SERVE_FRONTEND` 默认 `off`,所以开发与测试行为完全不变。 ```bash -# ---- 开发机:打包(内容白名单 + 体检,体检不过直接报错不产出包)---- -./scripts/deploy.sh --check # 只看会带上什么、排掉什么 +# ---- 开发机:数据集 → 打包(内容白名单 + 体检,体检不过直接报错不产出包)---- +.venv/bin/python scripts/seed.py doctor # 量一遍「开发库数据集 vs 当前代码」,只读 +.venv/bin/python scripts/seed.py export --fix # 补齐能力位/工作流键 → 体检 → 导出 scripts/seed/ +./scripts/deploy.sh --check # 只看会带上什么、排掉什么、数据能不能装 ./scripts/deploy.sh # → .local/deploy/nex_animation-<时间戳>.tar.gz ./scripts/deploy.sh --host deploy@10.0.0.8 \ --path /opt/nex_animation # 上传 + 远端 install + restart + status # ---- 服务器:落地 ---- tar -xzf nex_animation-*.tar.gz && cd nex_animation -./scripts/server.sh install # venv + 依赖 + npm build + 听写模型 + 建表灌种子数据 +./scripts/server.sh install # venv + 依赖 + npm build + 听写模型 + 体检 + 建表灌种子数据 ./scripts/server.sh start # 单进程 uvicorn,默认 :8000 ./scripts/server.sh verify # 图2 全 12 节点真实 HTTP + ffprobe 自检 ./scripts/server.sh status # 端口 / 健康 / 默认模型 / ComfyUI 连通性 / 数据边界 @@ -356,12 +358,20 @@ sudo ./scripts/server.sh systemd > /etc/systemd/system/nex-animation.service \ | 表结构:应用首启 `app/db.py:init_db()` 自己 `create_all` + 补列 | `media/` 产物库——换环境是「迁移」不是「部署」,单独 rsync | | 模型配置 `model_specs` + 加密主密钥 `model_secret_keys`(`scripts/seed/`) | 项目 / 分镜 / 任务等业务数据(开发库全是过程产物) | | 管理员账号 `users`(仅 `admin`,沿用导出的口令哈希) | `.env`——ComfyUI 地址、数据库串都是机器相关的 | -| 启动脚本与 `DEPLOY-INFO.txt`(包里有什么、服务器三步) | `backend/tests`、`reports/`、`docs/`、`models/`、`frontend/dist`(远端 build;`--with-dist` 可带上) | +| 启动脚本与 `DEPLOY-INFO.txt`(包里有什么、服务器三步、默认位是哪几个) | `backend/tests`、`reports/`、`.git/`、`models/`(听写权重远端下载)、`frontend/dist`(远端 build;`--with-dist` 可带上) | 主密钥跟包走是有意的:`model_specs.api_key` 是密文,换机器解不开就等于每台服务器都要重录一遍 各家 Key。代价是 `scripts/seed/` 属于**机密**——已 gitignore、随包走、包内是密文但主密钥也在里面, -按密钥对待。装完 `seed.py install` 会自检解密并打印当前默认模型;`--only-missing` 只补缺不覆盖 -已有配置,`--dry-run` 先看将要发生什么,主密钥冲突时区分「没有解密依赖 → 换」与「有依赖 → 中止等你决策」。 +按密钥对待。装完 `seed.py install` 会自检解密并打印当前默认模型;`install --only-missing` +(`seed.py` 与 `server.sh` 都收这个开关)只补缺、不覆盖同名配置,`--dry-run` 先看将要发生什么, +主密钥冲突时区分「没有解密依赖 → 换」与「有依赖 → 中止等你决策」。 + +**数据集与代码必须对得上**:`deploy.sh --check`、`deploy.sh` 打包、`seed.py install` 装库之前, +跑的都是同一个 `doctor`(列对得上表结构 / 适配器在代码里注册过 / `modes` 不比适配器声明的能力少 / +每类能力有且仅有一个默认 / 包里的密文能被包里的主密钥解开)。任何一条 error 就停住, +因为数据集旧的后果不是报错而是**功能在页面上凭空消失**——比如 `modes` 少一个 +`voice_instruct`,「音色描述直出」整块就不渲染,而 ComfyUI 侧看起来一切正常。 +修复动作永远在开发机:`seed.py doctor --fix`(只补缺,不动人工调过的值)→ 重新 `export --fix` → 重新打包。 > 服务器上没有 `whisper-cli` / 听写模型时,只有「台词听写质检」这一项不生效,其余流程照常; > `server.sh status` 会把它标成黄色告警而不是静默降级。`--no-asr` 可以跳过下载(465M 权重)。 diff --git a/backend/app/api/library.py b/backend/app/api/library.py index c3942a1..44199c3 100644 --- a/backend/app/api/library.py +++ b/backend/app/api/library.py @@ -337,7 +337,8 @@ def voice_reference_status(project_id: int, voice_id: int, db: Session = Depends deps.get_project(db, project_id) voice = deps.get_voice(db, project_id, voice_id) spec, _adapter = model_registry.resolve_voice(db, None, None) - return {"ok": True, "result": timbre.reference_status(db, voice, clone=timbre.clone_model(spec))} + return {"ok": True, "result": timbre.reference_status( + db, voice, clone=timbre.clone_model(spec), instruct=timbre.instruct_model(spec))} @router.delete("/api/projects/{project_id}/voices/{voice_id}") diff --git a/backend/app/api/platform.py b/backend/app/api/platform.py index 23305e5..f6225dc 100644 --- a/backend/app/api/platform.py +++ b/backend/app/api/platform.py @@ -966,7 +966,7 @@ def platform_reference_status(owner_type: str, asset_id: int, db: Session = Depe owner = _owner(db, owner_type, asset_id) spec, _adapter = model_registry.resolve_voice(db, None, None) return {"ok": True, "result": timbre.reference_status( - db, owner, clone=timbre.clone_model(spec))} + db, owner, clone=timbre.clone_model(spec), instruct=timbre.instruct_model(spec))} @router.post("/api/platform/{owner_type}/{asset_id}/generate") @@ -995,7 +995,7 @@ def generate_platform_asset(owner_type: str, asset_id: int, data: schemas.Platfo reference = timbre.resolve_reference( db, owner, explicit=str(storage.to_absolute(reference_media.path)) if reference_media else "", - clone=timbre.clone_model(spec)) + clone=timbre.clone_model(spec), instruct=timbre.instruct_model(spec)) if not reference["path"] and reference.get("source") == "none": raise HTTPException(409, reference["note"]) # 交给模型的是绝对路径(供应商要直接读文件);声线列里存 media_root 相对路径, @@ -1005,6 +1005,8 @@ def generate_platform_asset(owner_type: str, asset_id: int, data: schemas.Platfo text=text, voice_id=owner.name, provider_voice_id=reference_path, language=owner.language, speed=float((owner.tuning or {}).get("speed", 1.0)), seed=data.seed, params=(owner.tuning or {}).get("params") or {}, + # 没有参考音频时靠它出音,有参考音频时用它微调音色(见 comfy_assets.graph) + instruct_text=timbre.instruct_text_of(owner) if timbre.instruct_model(spec) else "", idempotency_key=f"platform-voice:{owner.id}:{asset_id}", label=f"PLATFORM-VOICE-{owner.id}", output_path=out, ) diff --git a/backend/app/api/system.py b/backend/app/api/system.py index d461906..3fa0401 100644 --- a/backend/app/api/system.py +++ b/backend/app/api/system.py @@ -33,7 +33,9 @@ MODEL_ADAPTERS = { "llm": ["mock_llm", "http_llm"], "video": ["mock", "http", "comfyui"], "image": ["mock_image", "http_image", "comfyui_image"], - "tts": ["mock", "http", "macos_say", "comfyui_tts"], + # 没有 macos_say:部署服务器是 Linux,没有系统 say 命令, + # 开发机上能用的本地嗓门不是产品能力。 + "tts": ["mock", "http", "comfyui_tts"], } diff --git a/backend/app/jobs/handlers.py b/backend/app/jobs/handlers.py index 0f97d9e..5645887 100644 --- a/backend/app/jobs/handlers.py +++ b/backend/app/jobs/handlers.py @@ -578,8 +578,10 @@ def _generate_voice_sample(db: Session, task: Task, project: Project, payload: d # 参考音频决定音色,「性别」对 zero-shot 克隆模型只是元数据: # 缺参考时必须换成同性别参考或明确报错,不能让男声悄悄用上模型默认的女声底模。 clone = timbre.clone_model(spec) + instruct = timbre.instruct_model(spec) reference = timbre.resolve_reference( - db, owner, explicit=reference_paths[0] if reference_paths else "", clone=clone) + db, owner, explicit=reference_paths[0] if reference_paths else "", + clone=clone, instruct=instruct) if not reference["path"] and reference.get("source") == "none": raise ProviderError(reference["note"], code="missing_reference_audio", retryable=False) request = VoiceGenRequest( @@ -588,7 +590,9 @@ def _generate_voice_sample(db: Session, task: Task, project: Project, payload: d language=owner.language, speed=float(tuning.get("speed", 1.0)), pitch=float(tuning.get("pitch", 1.0)), volume=float(tuning.get("volume", 1.0)), sample_rate=int(tuning.get("sample_rate", 24000)), seed=payload.get("seed"), - params=tuning.get("params") or {}, idempotency_key=f"asset-voice:{owner.id}", + params=tuning.get("params") or {}, + instruct_text=timbre.instruct_text_of(owner) if instruct else "", + idempotency_key=f"asset-voice:{owner.id}", label=f"VOICE-{owner.id}", output_path=out, ) dispatch.heartbeat(db, task, progress=0.3, note=f"调用 {spec.provider}/{spec.model} 生成试听") diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py index f0c57e7..12bec13 100644 --- a/backend/app/providers/base.py +++ b/backend/app/providers/base.py @@ -120,6 +120,9 @@ class VoiceGenRequest: idempotency_key: str = "" label: str = "" output_path: Path | None = None + #: 音色/情绪描述(声线的「音色描述」)。支持 voice_instruct 的模型在 + #: **没有参考音频**时靠它出音;有参考音频时用它做音色偏移与情绪控制。 + instruct_text: str = "" @dataclass diff --git a/backend/app/providers/comfy_assets.py b/backend/app/providers/comfy_assets.py index 558db6f..4532eb0 100644 --- a/backend/app/providers/comfy_assets.py +++ b/backend/app/providers/comfy_assets.py @@ -403,9 +403,11 @@ class ComfyTTSProvider(VoiceProviderAdapter): return self._name def capabilities(self, config: dict | None = None) -> Capabilities: - return Capabilities(modes=("voice", "voice_clone"), resolutions=(), aspect_ratios=(), + return Capabilities(modes=("voice", "voice_clone", "voice_instruct"), + resolutions=(), aspect_ratios=(), min_duration=0, max_duration=0, max_reference_images=1, - notes="IndexTTS2 本地推理;使用参考音频做 zero-shot 音色克隆") + notes="IndexTTS2 本地推理:有声线参考音频时做 zero-shot 音色克隆;" + "没有参考音频时用「音色描述」直接生成") _probe = _ProbeCache() @@ -442,8 +444,18 @@ class ComfyTTSProvider(VoiceProviderAdapter): except FatalProviderError: cls._probe.invalidate() + #: easy indexTTSGenerate(全参节点)的必填项。描述词模式必须走这个节点—— + #: ``easy indexTTSGenerateSimple`` 不接 emotions 输入,描述词写进去也是聋子。 + #: 数值取 ComfyUI-Index-TTS 节点自身的默认值,模型配置 generate_full_inputs 可逐项覆盖。 + FULL_DEFAULTS: dict[str, Any] = { + "do_sample": True, "temperature": 0.8, "top_p": 0.9, "top_k": 30, + "num_beams": 3, "repetition_penalty": 10.0, "length_penalty": 0.0, + "max_mel_tokens": 1815, "max_tokens_per_sentence": 120, "speech_speed": 1.0, + } + @staticmethod - def graph(request: VoiceGenRequest, config: dict, seed: int, reference: str) -> dict[str, Any]: + def graph(request: VoiceGenRequest, config: dict, seed: int, reference: str, + instruct: str = "") -> dict[str, Any]: loader = str(config.get("load_node") or "easy downloadIndexTTSAndLoadModel") generate = str(config.get("generate_node") or "easy indexTTSGenerateSimple") model_name = str(config.get("model") or "IndexTTS-2") @@ -460,7 +472,7 @@ class ComfyTTSProvider(VoiceProviderAdapter): "seed": seed, } generate_inputs.update(config.get("generate_inputs") or {}) - return { + graph: dict[str, Any] = { "1": {"class_type": loader, "inputs": load_inputs}, "2": {"class_type": "LoadAudio", "inputs": {"audio": reference}}, "3": {"class_type": generate, "inputs": generate_inputs}, @@ -468,6 +480,23 @@ class ComfyTTSProvider(VoiceProviderAdapter): "audio": ["3", 0], "filename_prefix": str(config.get("filename_prefix") or "audio/nex")}}, } + if instruct: + full: dict[str, Any] = dict(ComfyTTSProvider.FULL_DEFAULTS) + full.update(config.get("generate_full_inputs") or {}) + full.update({"indextts_model": ["1", 0], "text": request.text, + "unload_model": False, "seed": seed}) + # 有声线参考时克隆照旧:描述词只做音色偏移与情绪,不该把克隆来的嗓子丢掉。 + if reference: + full["reference_audio"] = ["2", 0] + graph["3"] = {"class_type": str(config.get("generate_full_node") or "easy indexTTSGenerate"), + "inputs": full} + graph["5"] = {"class_type": str(config.get("instruct_node") or "easy indexTTSEmotionText"), + "inputs": {"reference_audio": ["2", 0], + "voice_name": str(request.label or request.voice_id or "s1"), + "description": instruct, + "use_random": bool(config.get("instruct_random"))}} + graph["3"]["inputs"]["emotions"] = ["5", 0] + return graph def synthesize(self, request: VoiceGenRequest, config: dict | None = None) -> VoiceGenResult: cfg = dict(config or {}) @@ -489,9 +518,19 @@ class ComfyTTSProvider(VoiceProviderAdapter): # reference_source 会进配音记录:排查「全片都是同一个陌生人音色」时, # 必须能一眼看出这条配音用的是角色自己的参考音频还是模型默认参考。 reference_source = "voice_asset" if candidate else "model_default" + # 描述词直出:声线没有参考音频也能出音。emotion 节点的 reference_audio 是必填, + # 所以锚点仍取模型默认参考——音色由描述词决定(9200 实测:同一句台词, + # 「年幼小女孩」中位基频 241Hz、「低沉老年男性」189Hz,描述词真的在改音色)。 + instruct = str(request.instruct_text or (request.params or {}).get("instruct") or "").strip() + if not reference and instruct: + reference = configured_reference + reference_source = "instruct_anchor" + mode = "voice_clone" if candidate else ("voice_instruct" if instruct else "voice_default") if not reference: - raise FatalProviderError("IndexTTS2 需要参考音频:请给声线填写 provider_voice_id" - " 或给模型配置 reference_audio", code="missing_reference_audio") + raise FatalProviderError( + "IndexTTS2 既没有参考音频也没有音色描述:" + "请给声线上传参考音频(克隆),或填写「音色描述」(描述词直出)", + code="missing_reference_audio") seed = request.seed if request.seed is not None else int( hashlib.sha256((request.idempotency_key or request.text).encode()).hexdigest()[:12], 16 ) @@ -503,12 +542,17 @@ class ComfyTTSProvider(VoiceProviderAdapter): nodes = client.get(f"{base}/object_info", timeout=60) raise_for(nodes, context="读取 ComfyUI 节点清单") available_nodes = set((nodes.json() or {}).keys()) - required = {str(cfg.get("load_node") or "easy downloadIndexTTSAndLoadModel"), - str(cfg.get("generate_node") or "easy indexTTSGenerateSimple")} + required = {str(cfg.get("load_node") or "easy downloadIndexTTSAndLoadModel")} + if instruct: + required |= {str(cfg.get("generate_full_node") or "easy indexTTSGenerate"), + str(cfg.get("instruct_node") or "easy indexTTSEmotionText")} + else: + required.add(str(cfg.get("generate_node") or "easy indexTTSGenerateSimple")) missing = sorted(required - available_nodes) if missing: raise FatalProviderError( "ComfyUI 缺少 IndexTTS2 节点:" + "、".join(missing) + + ("(描述词模式还需要全参生成节点与音色描述节点)" if instruct else "") + "。请在 9200 实例安装并启用对应的 IndexTTS 自定义节点后重试", code="missing_comfy_node", ) @@ -524,7 +568,7 @@ class ComfyTTSProvider(VoiceProviderAdapter): # 覆盖模型默认参考音频,否则每次生成都会在 LoadAudio 校验阶段失败。 reference = configured_reference reference_source = "model_default" - graph = self.graph(request, cfg, int(seed), reference) + graph = self.graph(request, cfg, int(seed), reference, instruct) prompt_id = submit_graph(client, base, graph, stable_client_id(request.idempotency_key or request.label or request.text), idempotency_key=request.idempotency_key) @@ -554,12 +598,16 @@ class ComfyTTSProvider(VoiceProviderAdapter): metadata={"provider": "comfyui_tts", "base_url": base, "seed": seed, "reference_audio": reference, "reference_source": reference_source, + "mode": mode, "instruct_text": instruct, "tuning": tuning, "comfy_output": f"{item.get('subfolder', '')}/{item['filename']}".strip("/")}) def diagnose(self, config: dict | None = None) -> dict: return _diagnose(config or {}, required_nodes=( "easy downloadIndexTTSAndLoadModel", "easy indexTTSGenerateSimple", + # 描述词直出的两个节点一起查:缺了它们「填了音色描述却只会克隆」 + # 要等到第一次生成才暴露,太晚了。 + "easy indexTTSGenerate", "easy indexTTSEmotionText", "LoadAudio", "SaveAudio", )) diff --git a/backend/app/providers/macos_say.py b/backend/app/providers/macos_say.py deleted file mode 100644 index 36712a0..0000000 --- a/backend/app/providers/macos_say.py +++ /dev/null @@ -1,119 +0,0 @@ -"""macOS 系统语音适配器:把真实 TTS 能力接入统一配音流水线。 - -``say`` 是 macOS 自带命令,适合本地开发与离线成片。适配器统一输出单声道 -PCM WAV,并可对音色追加滤镜,例如铁血战士的外星语效果。 -""" -from __future__ import annotations - -import shutil -import subprocess -import tempfile -from pathlib import Path - -from app.config import settings -from app.providers.base import (FatalProviderError, VoiceGenRequest, VoiceGenResult, - VoiceProviderAdapter) -from app.services import ffmpeg as ff - - -# 低频化 + 金属共振 + 颗粒噪声;保持原始语速的同时明显降低音高。 -ALIEN_AUDIO_FILTER = ( - "asetrate=22050*0.62,aresample=22050,atempo=1.6129," - "flanger=delay=4:depth=3:regen=25:width=60:speed=0.4," - "aecho=0.8:0.7:40|90:0.35|0.2," - "acrusher=bits=8:mode=log:aa=1,lowpass=f=4800,tremolo=f=7:d=0.25" -) - - -class MacOSSayVoiceProvider(VoiceProviderAdapter): - """使用 macOS ``say`` 合成可听的真实语音。 - - 声线通过 ``provider_voice_id`` 指定,例如 ``Tingting``、``Meijia``、 - ``Sinji`` 或 ``Fred``。``request.params.effect = "alien"`` 会启用外星语 - 滤镜链。模型配置中的 ``voice`` 和 ``effect`` 可作为默认值。 - """ - - _name = "macos_say" - - @property - def name(self) -> str: - return self._name - - def is_available(self, config: dict | None = None) -> bool: - return bool(shutil.which("say")) and ff.available() - - def synthesize(self, request: VoiceGenRequest, config: dict | None = None) -> VoiceGenResult: - config = config or {} - say_bin = shutil.which("say") - if not say_bin: - raise FatalProviderError("当前系统没有 macOS say 命令,无法使用 macos_say 配音", - code="say_unavailable") - if not ff.available(): - raise FatalProviderError("本机缺少 ffmpeg/ffprobe,无法转换 say 音频", - code="ffmpeg_unavailable") - - text = (request.text or "").strip() - if not text: - raise FatalProviderError("配音文本为空", code="empty_text") - - voice = (request.provider_voice_id or config.get("voice") or "Tingting").strip() - if not voice: - raise FatalProviderError("未配置 macOS 系统声线 ID", code="missing_voice") - speed = max(0.5, min(float(request.speed or 1.0), 2.0)) - rate = max(80, min(420, int(round(180 * speed)))) - - out = Path(request.output_path or settings.media_root / "macos_say" / "voice.wav") - out.parent.mkdir(parents=True, exist_ok=True) - - with tempfile.TemporaryDirectory(prefix="nex-say-") as tmp: - raw = Path(tmp) / "speech.aiff" - say_cmd = [say_bin, "-v", voice, "-r", str(rate), "-o", str(raw), "--", text] - try: - proc = subprocess.run(say_cmd, capture_output=True, text=True, timeout=120) - except subprocess.TimeoutExpired as exc: - raise FatalProviderError("macOS say 合成超时", code="say_timeout") from exc - if proc.returncode != 0: - detail = (proc.stderr or proc.stdout or "未知错误").strip() - raise FatalProviderError( - f"macOS say 合成失败(声线 {voice}):{detail}", - code="say_failed", detail={"voice": voice, "stderr": detail}) - if not raw.exists() or raw.stat().st_size < 512: - raise FatalProviderError( - f"声线「{voice}」没有生成有效音频,可能尚未下载到本机", - code="say_empty_output", detail={"voice": voice}) - - filters = [] - effect = str(request.params.get("effect") or config.get("effect") or "").lower() - pitch = max(0.5, min(float(request.pitch or 1.0), 2.0)) - if effect == "alien": - filters.append(ALIEN_AUDIO_FILTER) - elif abs(pitch - 1.0) > 0.01: - sample_rate = max(8000, min(int(request.sample_rate or 24000), 48000)) - filters.extend([ - f"asetrate={sample_rate}*{pitch:.4f}", - f"aresample={sample_rate}", - f"atempo={1.0 / pitch:.4f}", - ]) - - cmd = [ff.FFMPEG, "-v", "error", "-y", "-i", str(raw), "-vn"] - if filters: - cmd += ["-af", ",".join(filters)] - cmd += ["-ar", str(max(8000, min(int(request.sample_rate or 24000), 48000))), - "-ac", "1", "-c:a", "pcm_s16le", str(out)] - ff.run(cmd, timeout=180) - - probed = ff.probe(out) - duration = float(probed.duration or 0.0) - if not out.exists() or out.stat().st_size < 512 or duration < 0.3: - raise FatalProviderError( - f"声线「{voice}」输出的音频过短({duration:.3f}s),请换用已下载的系统语音", - code="say_invalid_audio", detail={"voice": voice, "duration": duration}) - - return VoiceGenResult( - file_path=str(out), - duration_ms=int(round(duration * 1000)), - provider_job_id=f"macos-say-{abs(hash((voice, text, effect))) % 10**10}", - cost=0.0, - metadata={"provider": "macos_say", "voice": voice, "effect": effect, - "rate": rate, "duration": duration, "real": True}, - ) diff --git a/backend/app/providers/registry.py b/backend/app/providers/registry.py index b1c622d..e611f77 100644 --- a/backend/app/providers/registry.py +++ b/backend/app/providers/registry.py @@ -14,7 +14,6 @@ from app.providers.comfy_assets import ComfyImageProvider, ComfyTTSProvider from app.providers.comfyui import ComfyVideoProvider from app.providers.http_generic import HttpVideoProvider, HttpVoiceProvider from app.providers.llm import HttpLLMProvider, MockLLMProvider -from app.providers.macos_say import MacOSSayVoiceProvider from app.providers.mock import MockVideoProvider, MockVoiceProvider from app.providers.studio_assets import (HttpImageProvider, HttpTextProvider, MockImageProvider, MockTextProvider) @@ -25,7 +24,6 @@ _VIDEO_ADAPTERS: dict[str, VideoProviderAdapter] = {"mock": MockVideoProvider(), _VOICE_ADAPTERS: dict[str, VoiceProviderAdapter] = { "mock": MockVoiceProvider(), "http": HttpVoiceProvider(), - "macos_say": MacOSSayVoiceProvider(), "comfyui_tts": ComfyTTSProvider(), } _IMAGE_ADAPTERS: dict[str, ImageProviderAdapter] = { @@ -320,7 +318,10 @@ def comfyui_tts_config(**overrides) -> dict: "download_from": "modelscope", "load_node": "easy downloadIndexTTSAndLoadModel", "generate_node": "easy indexTTSGenerateSimple", - "reference_audio": "nex_tts_reference.wav", + # 描述词直出(voice_instruct)走全参节点 + 音色描述节点; + # 锚点音频沿用 reference_audio:它只当地音,音色由声线的「音色描述」决定。 + "generate_full_node": "easy indexTTSGenerate", + "instruct_node": "easy indexTTSEmotionText", "input_subfolder": "nex_animation", "filename_prefix": "audio/nex", "poll_interval": 1, @@ -365,12 +366,42 @@ def _has_real_spec(db: Session, media_type: str) -> bool: ModelSpec.enabled.is_(True)).first() is not None +def _retire_macos_say(db: Session) -> bool: + """删掉历史遗留的 macOS 系统语音模型,返回是否改动过数据库。 + + 部署目标是 Linux,`say` 命令在服务器上不存在:这台开发机上登记过 macos-say + 的话,模型列表里会永远挂着一个在生产环境不可用的「mac 本地」音频模型。 + 以前靠 ``is_available()`` 兜着——本机有 say 就登记,看着没错, + 实际是把开发机的能力当成了产品能力。现在直接不登记,并把旧行删干净。 + """ + from app.models import ModelSpec + + rows = list(db.scalars(select(ModelSpec).where(ModelSpec.provider == "macos-say")).all()) + if not rows: + return False + media_types = {row.media_type for row in rows} + for row in rows: + db.delete(row) + db.flush() + # 被删的行里如果有默认模型,那个能力就没人兜底了:把剩下最早的真实模型提为默认。 + for media_type in media_types: + if db.query(ModelSpec.id).filter(ModelSpec.media_type == media_type, + ModelSpec.is_default.is_(True)).first(): + continue + successor = db.scalars(select(ModelSpec).where(ModelSpec.media_type == media_type) + .order_by(ModelSpec.is_default.desc(), ModelSpec.id)).first() + if successor is not None: + successor.is_default = True + db.flush() + return True + + def seed_specs(db: Session) -> None: """首次启动时注册 mock 模型;如配置了自定义 HTTP 供应商则一并注册。""" from app.models import ModelSpec existing = {(s.provider, s.model, s.media_type) for s in db.scalars(select(ModelSpec)).all()} - added = _normalize_llm_specs(db) + added = _normalize_llm_specs(db) or _retire_macos_say(db) def ensure(**kwargs) -> None: nonlocal added @@ -416,16 +447,6 @@ def seed_specs(db: Session) -> None: modes=["chat", "prompt"], min_duration=0, max_duration=0, max_reference_images=0, currency=settings.cost_currency, notes="离线大模型:仅验证结构化文本与提示词链路;图像生成请使用独立图像模型") - say_provider = MacOSSayVoiceProvider() - if say_provider.is_available(): - ensure(provider="macos-say", model="system-voice", - display_name="macOS 系统语音(真实人声)", media_type="tts", - adapter="macos_say", is_default=settings.tts_provider == "macos-say", - max_concurrency=2, requests_per_minute=30, modes=["voice"], - price_per_1k_chars=0.0, currency=settings.cost_currency, - config_json={"voice": "Tingting"}, - notes="使用 macOS say 在本地合成真实语音;声线 ID 可填 Tingting、Meijia、" - "Sinji、Fred 等系统语音,params.effect=alien 可生成外星语效果") if settings.comfyui_base_url: ensure(provider="comfyui", model=settings.comfyui_model, display_name=f"ComfyUI · {settings.comfyui_model}", media_type="video", @@ -448,11 +469,13 @@ def seed_specs(db: Session) -> None: ensure(provider="comfyui", model="indextts2", display_name="IndexTTS2(本地音色克隆)", media_type="tts", adapter="comfyui_tts", is_default=False, - max_concurrency=1, requests_per_minute=12, modes=["voice", "voice_clone"], + max_concurrency=1, requests_per_minute=12, + modes=["voice", "voice_clone", "voice_instruct"], min_duration=0, max_duration=0, max_reference_images=1, price_per_1k_chars=0.0, currency=settings.cost_currency, config_json=comfyui_tts_config(), - notes="9200 本地 IndexTTS2;通过声线的参考音频 ID / 本地音频文件做音色克隆") + notes="9200 本地 IndexTTS2;声线带参考音频时做 zero-shot 音色克隆," + "没有参考音频时按声线的「音色描述」直接生成语音") if settings.video_api_base_url: ensure(provider="http", model=settings.video_model or "custom-video", display_name="自定义 HTTP 视频供应商", media_type="video", adapter="http", diff --git a/backend/app/services/audio.py b/backend/app/services/audio.py index 8a9ca5f..152f2ba 100644 --- a/backend/app/services/audio.py +++ b/backend/app/services/audio.py @@ -14,7 +14,7 @@ from app.config import settings from app.models import Character, Project, Shot, VoiceAsset from app.providers import registry as model_registry from app.providers.base import VoiceGenRequest -from app.services import costing, events, storage +from app.services import costing, events, storage, timbre from app.services import ffmpeg as ff # 多角色对白逐行合成时,换人之间留的静音(毫秒) @@ -571,6 +571,9 @@ def _tts_produce(db: Session, project: Project, shot: Shot, text: str, voice: Vo pitch=float(tuning.get("pitch", 1.0)), volume=float(tuning.get("volume", 1.0)), sample_rate=int(tuning.get("sample_rate", 24000)), seed=(int(shot.seed or 0) + index), params={**(tuning.get("params") or {}), "shot_code": shot.code}, + # 描述词直出:声线没录过音也能配上音(音色由「音色描述」决定) + instruct_text=(timbre.instruct_text_of(voice) + if voice is not None and timbre.instruct_model(spec) else ""), idempotency_key=f"voice-{shot.id}-{index}", label=f"SHOT-{shot.order_index:02d}-{index + 1:02d}", output_path=out) result = model_registry.generate_voice(db, request, spec.provider, spec.model) diff --git a/backend/app/services/dispatch.py b/backend/app/services/dispatch.py index eae8c43..2438566 100644 --- a/backend/app/services/dispatch.py +++ b/backend/app/services/dispatch.py @@ -273,7 +273,7 @@ def new_voiceover_task(db: Session, project: Project, shot: Shot, *, priority: i from app.services import audio # 任务必须记录镜头实际绑定的声线供应商。之前一律按全局默认 TTS 落库, - # 会出现任务显示 mock、产物却由 macos-say 生成的可观测性断层。 + # 会出现任务显示 mock、产物却由 IndexTTS2 生成的可观测性断层。 voice = audio.resolve_voice_asset(db, project, shot) spec, _ = audio.resolve_tts_spec(db, voice) text = (shot.dialogue or "").strip() diff --git a/backend/app/services/timbre.py b/backend/app/services/timbre.py index cb8e041..536ceaa 100644 --- a/backend/app/services/timbre.py +++ b/backend/app/services/timbre.py @@ -6,10 +6,13 @@ 生成时回退到模型配置里的那一段默认参考(女声),出来的试听样本是女声, 界面却全绿、什么都不说。 -这里做两件事: +这里做三件事: 1. ``resolve_reference()``:生成前按性别挑一段**真正能决定音色**的参考音频, 挑不到就明确报错,绝不静默用默认嗓; -2. ``check_voice()``:生成后量一次中位基频,与设定性别比对, + 例外是 ``voice_instruct`` 模型(IndexTTS2 的描述词直出): + 声线填了「音色描述」就走 ``source="instruct"``,没有参考音频也算可用; +2. ``instruct_text_of()``:把声线的音色描述取出来交给适配器; +3. ``check_voice()``:生成后量一次中位基频,与设定性别比对, 结果写进素材 metadata,界面能直接看到「音色与设定不符」。 """ from __future__ import annotations @@ -249,6 +252,23 @@ def clone_model(spec) -> bool: return bool((getattr(spec, "config_json", None) or {}).get("clone_reference")) +def instruct_model(spec) -> bool: + """这个 TTS 能否只靠「音色描述」出音(声线没有参考音频也算可用)。 + + IndexTTS2 除了 zero-shot 克隆还接受文本描述控制音色与情绪,登记在 + ``modes`` 里的 ``voice_instruct`` 就是这个能力。以前只认参考音频, + 没录过音的角色一律生成不了试听,用户只能先随便塞一段音频凑数。 + """ + if spec is None: + return False + return "voice_instruct" in (getattr(spec, "modes", None) or []) + + +def instruct_text_of(owner) -> str: + """声线/平台声线的「音色描述」;没有返回空串。""" + return str(getattr(owner, "description", "") or "").strip() + + def audio_path_or_empty(value: str | None) -> str: """声线记录里的 provider_voice_id 可能是显示名(如「低沉男声」),只有真音频路径才算数。""" from app.services.audio import AUDIO_SUFFIXES @@ -402,7 +422,8 @@ def _same_gender_voices(db: Session, owner, gender: str) -> list: return sorted(peers, key=lambda row: (not getattr(row, "is_default", False), -row.id)) -def resolve_reference(db: Session, owner, *, explicit: str = "", clone: bool = True) -> dict: +def resolve_reference(db: Session, owner, *, explicit: str = "", clone: bool = True, + instruct: bool = False) -> dict: """挑一段真正能决定音色的参考音频,并说明它是谁。 优先级:人工指定 → 本声线自己的参考 → 引入自的平台母版 → **同性别的其他声线** → 模型默认。 @@ -411,11 +432,15 @@ def resolve_reference(db: Session, owner, *, explicit: str = "", clone: bool = T ``clone=False`` 表示这个 TTS 走云端音色库(provider_voice_id 是音色 ID,不是参考音频), 此时没有本地参考音频不算问题,交回供应商音色 ID 处理。 + + ``instruct=True`` 表示这个模型支持「音色描述直出」(``voice_instruct``): + 找不到任何参考音频时不再报错,改成按声线自己填的音色描述生成(``source="instruct"``)。 + 参考音频仍然优先——克隆出来的嗓子比描述词精确得多。 """ chosen = audio_path_or_empty(explicit) if not clone: # 云端音色库靠 provider_voice_id 选音色,本地参考**不能**顶替它: - # 把 WAV 路径当音色 ID 发过去,macos_say 之类会直接拿到一个不存在的嗓门。 + # 把 WAV 路径当音色 ID 发过去,供应商会直接拿到一个不存在的嗓门。 # 但用户手动挂上来的参考仍然记进 attached,供素材元数据回溯用了哪段音频。 return {"path": "", "attached": chosen, "source": "provider_voice", "note": ("该模型用供应商音色 ID 选音色,本地参考音频只作记录、不决定音色" @@ -456,6 +481,11 @@ def resolve_reference(db: Session, owner, *, explicit: str = "", clone: bool = T if bound_generated else f"本声线还没有参考音频,本次以同性别声线「{sibling.name}」的音色为底生成;" f"想固定成自己的嗓子,请上传一段{GENDER_LABELS[gender]}参考音频")} + if instruct and instruct_text_of(owner): + return {"path": "", "source": "instruct", "bound_generated": bound_generated, + "note": f"没有{GENDER_LABELS[gender]}参考音频可用,本次按声线的「音色描述」生成" + f"(IndexTTS2 描述词直出);想要更稳的嗓子," + f"仍可上传一段{GENDER_LABELS[gender]}真人参考音频"} return {"path": "", "source": "none", "bound_generated": bound_generated, "note": (_AI_BOUND_NOTE + f"库里也没有任何{GENDER_LABELS[gender]}参考音频可用:" f"请上传一段{GENDER_LABELS[gender]}真人参考音频再生成" @@ -463,15 +493,18 @@ def resolve_reference(db: Session, owner, *, explicit: str = "", clone: bool = T f"库里没有任何{GENDER_LABELS[gender]}参考音频可用:" f"请上传一段{GENDER_LABELS[gender]}参考音频再生成," f"否则会拿到与设定不符的音色")} + if instruct and instruct_text_of(owner): + return {"path": "", "source": "instruct", "bound_generated": bound_generated, + "note": "本次按声线的「音色描述」生成:模型默认参考音频只作底声,音色由描述词决定"} return {"path": "", "source": "model_default", "bound_generated": bound_generated, "note": (_AI_BOUND_NOTE + "本次使用模型配置的默认参考音频(音色不受表单控制)" if bound_generated else "声线未限定性别,本次使用模型配置的默认参考音频(音色不受表单控制)")} -def reference_status(db: Session, owner, *, clone: bool = True) -> dict: +def reference_status(db: Session, owner, *, clone: bool = True, instruct: bool = False) -> dict: """给界面用的预告:现在点「生成」会拿到谁的嗓子,值不值得先传参考音频。""" - resolved = resolve_reference(db, owner, clone=clone) + resolved = resolve_reference(db, owner, clone=clone, instruct=instruct) gender = str(getattr(owner, "gender", "") or "").strip() return { "gender": gender, @@ -481,5 +514,9 @@ def reference_status(db: Session, owner, *, clone: bool = True) -> dict: "borrowed": resolved["source"] in ("gender_sibling", "platform_master"), # 用户视角的「我明明传过声音」:绑定的是 AI 样本时界面必须说清楚为什么不算 "bound_generated": bool(resolved.get("bound_generated")), - "ready": (bool(resolved["path"]) or resolved["source"] in ("model_default", "provider_voice")), + "ready": (bool(resolved["path"]) + or resolved["source"] in ("model_default", "provider_voice", "instruct")), + # 界面据此区分「克隆」与「描述词直出」两种音色来源,别把描述词画成「没有音色」 + "instruct": resolved["source"] == "instruct", + "instruct_text": instruct_text_of(owner) if instruct else "", } diff --git a/backend/tests/test_index_tts_instruct.py b/backend/tests/test_index_tts_instruct.py new file mode 100644 index 0000000..fc2c941 --- /dev/null +++ b/backend/tests/test_index_tts_instruct.py @@ -0,0 +1,190 @@ +"""回归用例:音频模型不许有「mac 本地」,IndexTTS2 必须能用「音色描述」直出。 + +用户口径(2026-11 第六轮): +1. 部署服务器是 Linux,没有系统 ``say`` 命令——「mac 本地(macos-say)」这条音频模型 + 在这台机器上永远跑不通,登记在库里只会让人选中一个不可用的模型; +2. IndexTTS2 除了参考音频克隆,还要支持**直接用描述词生成语音**: + 只填「音色描述」(例:年幼小女孩、低沉老年男性)就该出对应的嗓子, + 而不是必须先上传一段参考音频。 + +实测依据(直连 10.100.53.199:9200 提交图验证): +``easy indexTTSGenerateSimple`` 没有 emotions 输入,描述词写进去不生效; +描述词必须走 ``easy indexTTSGenerate``(全参)+ ``easy indexTTSEmotionText`` +(reference_audio / voice_name / description / use_random → 输出 emotions)。 +同一句台词用两组描述词实测中位基频 241.4 Hz ↔ 189.2 Hz,音色确实在变。 +""" +from __future__ import annotations + +import importlib + +from app.models import ModelSpec, PlatformVoice +from app.providers.base import VoiceGenRequest +from app.providers.comfy_assets import ComfyTTSProvider +from app.services import timbre + + +CFG = {"base_url": "http://127.0.0.1:9200"} + + +def _req(**kw) -> VoiceGenRequest: + base = dict(text="你好,我是这条声线。", label="声线A", output_path="/tmp/x.wav", + provider_voice_id="") + base.update(kw) + return VoiceGenRequest(**base) + + +# ------------------------------------------------------------------ 描述词图 +def test_instruct_switches_graph_to_full_node_with_emotion(): + """填了音色描述 → 换全参生成节点,并挂上描述词节点(Simple 节点不接 emotions)。""" + graph = ComfyTTSProvider.graph(_req(), CFG, 1234, "nex_animation/ref.wav", + instruct="低沉老年男性") + assert graph["3"]["class_type"] == "easy indexTTSGenerate" + assert graph["5"]["class_type"] == "easy indexTTSEmotionText" + assert graph["5"]["inputs"]["description"] == "低沉老年男性" + assert graph["5"]["inputs"]["reference_audio"] == ["2", 0] + assert graph["3"]["inputs"]["emotions"] == ["5", 0] + # 全参节点的必填项一个都不能少,缺一项 ComfyUI 会在提交阶段直接拒绝整张图 + for key, value in ComfyTTSProvider.FULL_DEFAULTS.items(): + assert graph["3"]["inputs"][key] == value, key + assert graph["3"]["inputs"]["text"] == "你好,我是这条声线。" + assert graph["3"]["inputs"]["seed"] == 1234 + + +def test_instruct_keeps_clone_reference_when_present(): + """有真人参考音频时克隆优先:描述词只做偏移,不该把克隆来的嗓子丢掉。""" + graph = ComfyTTSProvider.graph(_req(), CFG, 7, "nex_animation/ref.wav", + instruct="年幼小女孩") + assert graph["3"]["inputs"]["reference_audio"] == ["2", 0] + + +def test_description_only_mode_still_loads_the_anchor_audio(): + """只填描述词时仍要一段底声锚点:ComfyUI-Index-TTS 的 emotion 节点 + ``reference_audio`` 是必填项,缺了它整张图提交就被拒。 + + 锚点用模型配置的默认参考音频,音色由描述词决定——实测同一句台词两组描述词 + 中位基频 241.4 Hz ↔ 189.2 Hz,锚点相同而嗓子不同。 + """ + graph = ComfyTTSProvider.graph(_req(), CFG, 7, "nex_default.wav", + instruct="年幼小女孩") + assert graph["2"]["inputs"]["audio"] == "nex_default.wav" + assert graph["5"]["inputs"]["reference_audio"] == ["2", 0] + assert graph["3"]["inputs"]["reference_audio"] == ["2", 0] + assert graph["3"]["inputs"]["emotions"] == ["5", 0] + + +def test_plain_clone_graph_is_unchanged(): + """没填描述词时仍是原来的 zero-shot 克隆图,老行为不受影响。""" + graph = ComfyTTSProvider.graph(_req(), CFG, 7, "nex_animation/ref.wav", instruct="") + assert graph["3"]["class_type"] == "easy indexTTSGenerateSimple" + assert "emotions" not in graph["3"]["inputs"] + assert "5" not in graph + + +def test_capabilities_declare_voice_instruct(): + caps = ComfyTTSProvider().capabilities(CFG) + assert "voice_instruct" in caps.modes + assert "voice_clone" in caps.modes + + +# ------------------------------------------------------------------ mac 本地下线 +def test_macos_say_adapter_is_gone(): + assert importlib.util.find_spec("app.providers.macos_say") is None + + +def test_registry_no_longer_seeds_macos_say(db): + from app.providers import registry + + registry.seed_specs(db) + db.commit() + assert db.query(ModelSpec).filter(ModelSpec.provider == "macos-say").count() == 0 + rows = list(db.query(ModelSpec).filter(ModelSpec.media_type == "tts").all()) + assert rows, "音频模型整类不能因为下线 macos-say 而被清空" + assert "macos-say" not in {r.provider for r in rows} + indextts = [r for r in rows if r.adapter == "comfyui_tts"] + assert indextts, "IndexTTS2 必须仍在册" + assert "voice_instruct" in (indextts[0].modes or []) + + +def test_legacy_macos_say_row_is_retired_and_default_promoted(db): + """库里已经登记过 macos-say(开发机残留):启动时删掉,默认值顺延给后继模型。""" + from app.providers import registry + + stale = ModelSpec(media_type="tts", provider="macos-say", adapter="macos_say", + display_name="mac 本地", model="system-voice", + model_code="macos-say/system-voice", is_default=True, enabled=True) + keeper = ModelSpec(media_type="tts", provider="comfyui_tts", adapter="comfyui_tts", + display_name="IndexTTS2", model="IndexTTS-2", + model_code="comfyui/indextts2", is_default=False, enabled=True, + modes=["voice", "voice_clone", "voice_instruct"]) + db.add_all([stale, keeper]) + db.commit() + + assert registry._retire_macos_say(db) is True + db.commit() + assert db.query(ModelSpec).filter(ModelSpec.provider == "macos-say").count() == 0 + db.refresh(keeper) + assert keeper.is_default is True + # 幂等:第二次没有 macos-say 行可删,不再动默认标记 + assert registry._retire_macos_say(db) is False + + +# ------------------------------------------------------------------ 音色解析 +def test_instruct_model_is_detected_from_modes(): + spec = ModelSpec(media_type="tts", provider="comfyui_tts", adapter="comfyui_tts", + display_name="IndexTTS2", model="IndexTTS-2", + modes=["voice", "voice_clone", "voice_instruct"]) + assert timbre.instruct_model(spec) is True + clone_only = ModelSpec(media_type="tts", provider="comfyui_tts", adapter="comfyui_tts", + display_name="只认参考音频的模型", model="x", + modes=["voice", "voice_clone"]) + assert timbre.instruct_model(clone_only) is False + + +def test_description_only_voice_resolves_to_instruct(db): + """没参考音频、没性别的声线:以前只能拿模型默认参考(音色不受控),现在走描述词。""" + voice = PlatformVoice(name="描述词声线", code="instruct-only", gender="", + description="年幼小女孩,音色清亮,语速偏快", + is_active=True, is_default=False, provider_voice_id="") + db.add(voice) + db.commit() + + assert timbre.instruct_text_of(voice) == "年幼小女孩,音色清亮,语速偏快" + resolved = timbre.resolve_reference(db, voice, instruct=True) + assert resolved["source"] == "instruct" + assert resolved["path"] == "" + assert "音色描述" in resolved["note"] + + status = timbre.reference_status(db, voice, instruct=True) + assert status["instruct"] is True + assert status["ready"] is True + assert status["instruct_text"] == "年幼小女孩,音色清亮,语速偏快" + + # 关掉描述词能力(换回只认参考音频的模型)时,预告回到原来的口径 + plain = timbre.reference_status(db, voice, instruct=False) + assert plain["instruct"] is False + assert plain["source"] == "model_default" + + +def test_instruct_needs_a_description(db): + """描述词是空的:不能谎称「按描述词生成」,仍按模型默认参考处理。""" + voice = PlatformVoice(name="空描述", code="instruct-empty", gender="", + description="", is_active=True, is_default=False, + provider_voice_id="") + db.add(voice) + db.commit() + assert timbre.resolve_reference(db, voice, instruct=True)["source"] == "model_default" + + +def test_reference_audio_still_wins_over_description(db, tmp_path): + """参考音频优先于描述词:克隆出来的嗓子比描述词精确得多。""" + anchor = tmp_path / "anchor.wav" + anchor.write_bytes(b"RIFF" + b"\0" * 40) + voice = PlatformVoice(name="有参考", code="instruct-ref", gender="male", + description="低沉老年男性", is_active=True, is_default=False, + provider_voice_id=str(anchor)) + db.add(voice) + db.commit() + + resolved = timbre.resolve_reference(db, voice, instruct=True) + assert resolved["source"] == "self" + assert resolved["path"] == str(anchor) diff --git a/backend/tests/test_macos_say.py b/backend/tests/test_macos_say.py deleted file mode 100644 index a4f05a4..0000000 --- a/backend/tests/test_macos_say.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import shutil - -import pytest - -from app.providers.base import FatalProviderError, VoiceGenRequest -from app.providers.macos_say import ALIEN_AUDIO_FILTER, MacOSSayVoiceProvider -from app.services import ffmpeg as ff - - -pytestmark = pytest.mark.skipif( - not shutil.which("say") or not ff.available(), - reason="需要 macOS say 与 ffmpeg", -) - - -def test_macos_say_generates_real_chinese_speech(tmp_path): - output = tmp_path / "waiter.wav" - result = MacOSSayVoiceProvider().synthesize( - VoiceGenRequest( - text="先生您好,今晚的头道菜是炭烤战斧牛排。", - provider_voice_id="Tingting", - language="zh-CN", - output_path=output, - ) - ) - - assert output.exists() - assert result.duration_ms >= 1000 - assert result.metadata["real"] is True - assert ff.probe(output).has_audio - - -def test_macos_say_alien_effect_produces_audible_audio(tmp_path): - output = tmp_path / "predator.wav" - result = MacOSSayVoiceProvider().synthesize( - VoiceGenRequest( - text="Sssth. Ka-laa, tuuk. Hsss, raa-ka-ta.", - provider_voice_id="Fred", - language="en", - params={"effect": "alien"}, - output_path=output, - ) - ) - - assert output.exists() - assert result.duration_ms >= 1000 - assert result.metadata["effect"] == "alien" - assert "acrusher" in ALIEN_AUDIO_FILTER - - -def test_macos_say_rejects_empty_text(tmp_path): - with pytest.raises(FatalProviderError, match="配音文本为空"): - MacOSSayVoiceProvider().synthesize( - VoiceGenRequest(text=" ", provider_voice_id="Tingting", output_path=tmp_path / "empty.wav") - ) - - -def test_voiceover_task_records_the_bound_macos_provider(api): - pid = api.project() - voice = api.post(f"/api/projects/{pid}/voices", json={ - "name": "真实中文女声", - "provider": "macos-say", - "provider_voice_id": "Tingting", - "language": "zh-CN", - })["result"] - shot = api.post(f"/api/projects/{pid}/shots", json={ - "title": "报菜名", "description": "服务员端上菜", "duration": 5, - "dialogue": "您的牛排,请慢用。", "voice_asset_id": voice["id"], - })["result"] - - api.post(f"/api/projects/{pid}/voiceover", json={"shot_ids": [shot["id"]]}) - task = next(t for t in api.get(f"/api/tasks?project_id={pid}&kind=voiceover")["tasks"] - if t["shot_id"] == shot["id"]) - assert task["provider"] == "macos-say", task - assert task["model"] == "system-voice", task diff --git a/docs/UI规范.md b/docs/UI规范.md index a61f094..3827f41 100644 --- a/docs/UI规范.md +++ b/docs/UI规范.md @@ -42,7 +42,7 @@ | R13 | **布局 utility 压在 antd 组件根节点上,依赖三处层叠声明**:`app/providers.tsx` 的 ``;`styles/index.css` 里拆开 import **之前**的 `@layer theme, base, components, antd, utilities;`;以及 `@import "tailwindcss/preflight.css" layer(antd)`——**preflight 必须与 antd 同层**,写回 `layer(base)` 会被 antd 的 prepend 反超、全站复位反吃组件样式。两者任一被摘掉,`max-md:hidden` 这类 utility 会**静默失效**(tsc / build 全绿,只有浏览器能看出来)(详见 §1.1) | Tailwind v4 把 utility 放进 `@layer`,而 antd 默认注入的 CSS 未分层;按 CSS 层叠规则「未分层永远赢过分层」, antd 会把我们的布局 utility 全量盖掉 | | R14 | **按钮与浮层底部只用 antd 原生词汇**:`type` / `danger` / `size` / `icon` / `shape` / `loading`,自造 `variant` 方言已废;浮层底部一律「辅助 → 取消 → 主行动」右对齐、整条 md 档、最多一颗 `type="primary"`(详见 §5.1、§10)。裸 `