"""数据库连接池:长命会话不得把别人挡在池口外。 真实事故:本项目用的是文件 SQLite,SQLAlchemy 默认 QueuePool 只有 5 常驻 + 10 溢出。 而内联 worker 一条会话要跨过几分钟的供应商调用、LLM 总控台一条会话能握几个小时, 15 条连接被这类长会话占满之后,后来者在池口排队 30 秒抛 ``QueuePool limit of size 5 overflow 10 reached``, `handle_generate_shot` 把它当成生成失败,连烧三次重试——两条已经排到 GPU 前的 视频就这么被判了死刑,而任务日志里连「数据库」三个字都没有。 """ from __future__ import annotations import socket import subprocess import sys import time from pathlib import Path from datetime import timedelta from sqlalchemy import text from sqlalchemy.engine.url import make_url from app.config import settings from app.db import (_build_engine, _pool_kwargs, open_sessions, pool_status, SessionLocal, session_trace_enabled) def _queued_shot_task(api, pid, script_text, make_image_png, colour): api.post(f"/api/projects/{pid}/import", data={"script_text": script_text, "default_duration": 2}, files=[("images", ("s01.png", make_image_png(480, 270, colour), "image/png"))]) api.post(f"/api/projects/{pid}/tasks", json={"with_voiceover": False}) return api.get("/api/tasks", params={"project_id": pid, "kind": "generate_shot"})["tasks"][0]["id"] def _make_zombie(task_id, *, attempts, dead_process=True, stale_seconds=0): """把任务伪装成「上一代进程遗留的 running」(可选:进程已消失 / 仅心跳超时)。""" import socket from app.models import Task, now_utc if dead_process: child = subprocess.Popen([sys.executable or "/bin/true", "-c", "pass"]) child.wait() # 一个确实已经消失的 pid worker_id = f"{socket.gethostname()}:{child.pid}" else: worker_id = "inline-worker-without-a-pid" # 解析不出 pid → 只能靠心跳超时兜底 session = SessionLocal() try: task = session.get(Task, task_id) task.status, task.attempts, task.max_attempts = "running", attempts, 3 task.execution_backend, task.worker_id = "inline", worker_id task.started_at = now_utc() task.heartbeat_at = now_utc() - timedelta(seconds=stale_seconds) session.commit() from app.services import dispatch recovered = dispatch.resume_interrupted(session) row = session.get(Task, task_id) return {"recovered": recovered, "status": row.status, "attempts": row.attempts, "checkpoint": dict(row.checkpoint_json or {})} finally: session.close() # ---------------------------------------------------------------- 连接池选型 def test_file_sqlite_uses_an_unbounded_pool(tmp_path): """文件库不再走有上限的 QueuePool;内存库仍必须共享同一条连接。""" assert _pool_kwargs(f"sqlite:///{tmp_path}/a.db") == {"poolclass": type(_build_engine(f"sqlite:///{tmp_path}/a.db").pool)} assert _pool_kwargs("sqlite:///:memory:")["pool_pre_ping"] is True def test_postgres_keeps_a_real_pool(): """Postgres 建连是 TCP+认证,必须继续池化——别把这条一起改掉。""" kwargs = _pool_kwargs("postgresql+psycopg://u:p@localhost:5432/db") assert kwargs["pool_size"] == 10 and kwargs["max_overflow"] == 20 and kwargs["pool_pre_ping"] is True def test_sessions_beyond_the_old_5_plus_10_limit_coexist(): """同时借出比旧上限(15)更多的会话:不排队、不超时。 自检库同样是文件 SQLite,所以这里直接借真引擎。旧实现下第 16 条要等满 pool_timeout(30 秒)才抛异常,这条用例会以「等待过久 + 异常」收场。 """ held = [] started = time.monotonic() try: for _ in range(25): session = SessionLocal() held.append(session) session.execute(text("select 1")) assert len(held) == 25 finally: for session in held: session.close() assert time.monotonic() - started < 10, "借会话不该排队等池子" def test_writes_still_serialise_correctly(): """NullPool 只是不复用连接,写事务的可见性语义必须原样保留。""" from app.models import Project name = "池子改造后的项目" session = SessionLocal() try: session.add(Project(name=name)) session.commit() finally: session.close() reader = SessionLocal() try: assert reader.query(Project).filter(Project.name == name).count() == 1 finally: reader.close() def test_pool_status_is_exposed_on_health(api): """/health 要能直接回答「是不是又在池口排队」。""" pool = api.get("/health")["database_pool"] assert pool["class"] in {"NullPool", "QueuePool", "StaticPool"} if session_trace_enabled(): assert isinstance(open_sessions(), list) assert pool.get("open_sessions", 0) >= 1 # 这次请求自己就占着一条 assert Path(make_url(settings.database_url).database).exists() # ---------------------------------------------------------------- 尝试次数账本 def test_reclaimed_attempt_is_refunded(api, studio, script_text, make_image_png): """热重载打断的那一次尝试不能算进重试额度。 attempts 记的是「开始执行了几次」,重试额度保护的是「模型侧失败几次」。 开发时反复改代码热重载会把前者刷爆:本机一个任务什么都没失败就跑到 6/3, 之后任何一个瞬时抖动都会直接把它判死。 """ task_id = _queued_shot_task(api, studio["project_id"], script_text, make_image_png, "0x226699") result = _make_zombie(task_id, attempts=3) assert result["recovered"]["count"] == 1 assert result["checkpoint"]["recovered_from"] == "dead_runner" assert result["checkpoint"]["attempt_refunded"] is True assert result["attempts"] == 2, "被打断的那次尝试要退还" assert result["status"] == "queued" def test_stale_running_attempt_is_not_refunded(api, studio, script_text, make_image_png): """心跳超时可能真的还在别处跑着,这种不退还,免得崩坏的任务无限循环。""" task_id = _queued_shot_task(api, studio["project_id"], script_text, make_image_png, "0x993355") result = _make_zombie(task_id, attempts=2, dead_process=False, stale_seconds=30) assert result["checkpoint"]["recovered_from"] == "stale_running" assert "attempt_refunded" not in result["checkpoint"] assert result["attempts"] == 2 def test_refund_leaves_a_working_retry_budget(api, studio, script_text, make_image_png): """退还不会把兜底改松:真失败照样按剩余额度重试,用尽后落败。""" pid = studio["project_id"] task_id = _queued_shot_task(api, pid, script_text, make_image_png, "0x117755") spec = next(r for r in api.get("/api/providers")["providers"] if r["provider"] == "mock" and r["media_type"] == "video") api.patch(f"/api/providers/{spec['id']}", json={"config": {"failure_rate": 1.0}}) # 被打断在 2/3:退还后仍留 1 次 → 真失败还能重试一次,然后按额度落败 assert _make_zombie(task_id, attempts=2)["attempts"] == 1 api.drain(pid) # 断点续跑后已是 queued,直接派发 back = api.get(f"/api/tasks/{task_id}")["task"] assert back["status"] == "failed", back assert back["attempts"] == back["max_attempts"] == 3, back assert [h["code"] for h in back["error_history"]] == ["mock_injected_failure"] * 2