nex_animation/backend/tests/test_db_pool.py

170 lines
7.6 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""数据库连接池:长命会话不得把别人挡在池口外。
真实事故:本项目用的是文件 SQLite,SQLAlchemy 默认 QueuePool 只有 5 常驻 + 10 溢出。
而内联 worker 一条会话要跨过几分钟的供应商调用、LLM 总控台一条会话能握几个小时,
15 条连接被这类长会话占满之后,后来者在池口排队 30 秒抛
``QueuePool limit of size 5 overflow 10 reached``,
`handle_generate_shot` 把它当成生成失败,连烧三次重试——两条已经排到 GPU 前的
视频就这么被判了死刑,而任务日志里连「数据库」三个字都没有。
"""
from __future__ import annotations
import socket
import subprocess
import sys
import time
from pathlib import Path
from datetime import timedelta
from sqlalchemy import text
from sqlalchemy.engine.url import make_url
from app.config import settings
from app.db import (_build_engine, _pool_kwargs, open_sessions, pool_status, SessionLocal,
session_trace_enabled)
def _queued_shot_task(api, pid, script_text, make_image_png, colour):
api.post(f"/api/projects/{pid}/import",
data={"script_text": script_text, "default_duration": 2},
files=[("images", ("s01.png", make_image_png(480, 270, colour), "image/png"))])
api.post(f"/api/projects/{pid}/tasks", json={"with_voiceover": False})
return api.get("/api/tasks", params={"project_id": pid, "kind": "generate_shot"})["tasks"][0]["id"]
def _make_zombie(task_id, *, attempts, dead_process=True, stale_seconds=0):
"""把任务伪装成「上一代进程遗留的 running」(可选:进程已消失 / 仅心跳超时)。"""
import socket
from app.models import Task, now_utc
if dead_process:
child = subprocess.Popen([sys.executable or "/bin/true", "-c", "pass"])
child.wait() # 一个确实已经消失的 pid
worker_id = f"{socket.gethostname()}:{child.pid}"
else:
worker_id = "inline-worker-without-a-pid" # 解析不出 pid → 只能靠心跳超时兜底
session = SessionLocal()
try:
task = session.get(Task, task_id)
task.status, task.attempts, task.max_attempts = "running", attempts, 3
task.execution_backend, task.worker_id = "inline", worker_id
task.started_at = now_utc()
task.heartbeat_at = now_utc() - timedelta(seconds=stale_seconds)
session.commit()
from app.services import dispatch
recovered = dispatch.resume_interrupted(session)
row = session.get(Task, task_id)
return {"recovered": recovered, "status": row.status, "attempts": row.attempts,
"checkpoint": dict(row.checkpoint_json or {})}
finally:
session.close()
# ---------------------------------------------------------------- 连接池选型
def test_file_sqlite_uses_an_unbounded_pool(tmp_path):
"""文件库不再走有上限的 QueuePool;内存库仍必须共享同一条连接。"""
assert _pool_kwargs(f"sqlite:///{tmp_path}/a.db") == {"poolclass": type(_build_engine(f"sqlite:///{tmp_path}/a.db").pool)}
assert _pool_kwargs("sqlite:///:memory:")["pool_pre_ping"] is True
def test_postgres_keeps_a_real_pool():
"""Postgres 建连是 TCP+认证,必须继续池化——别把这条一起改掉。"""
kwargs = _pool_kwargs("postgresql+psycopg://u:p@localhost:5432/db")
assert kwargs["pool_size"] == 10 and kwargs["max_overflow"] == 20 and kwargs["pool_pre_ping"] is True
def test_sessions_beyond_the_old_5_plus_10_limit_coexist():
"""同时借出比旧上限(15)更多的会话:不排队、不超时。
自检库同样是文件 SQLite,所以这里直接借真引擎。旧实现下第 16 条要等满
pool_timeout(30 秒)才抛异常,这条用例会以「等待过久 + 异常」收场。
"""
held = []
started = time.monotonic()
try:
for _ in range(25):
session = SessionLocal()
held.append(session)
session.execute(text("select 1"))
assert len(held) == 25
finally:
for session in held:
session.close()
assert time.monotonic() - started < 10, "借会话不该排队等池子"
def test_writes_still_serialise_correctly():
"""NullPool 只是不复用连接,写事务的可见性语义必须原样保留。"""
from app.models import Project
name = "池子改造后的项目"
session = SessionLocal()
try:
session.add(Project(name=name))
session.commit()
finally:
session.close()
reader = SessionLocal()
try:
assert reader.query(Project).filter(Project.name == name).count() == 1
finally:
reader.close()
def test_pool_status_is_exposed_on_health(api):
"""/health 要能直接回答「是不是又在池口排队」。"""
pool = api.get("/health")["database_pool"]
assert pool["class"] in {"NullPool", "QueuePool", "StaticPool"}
if session_trace_enabled():
assert isinstance(open_sessions(), list)
assert pool.get("open_sessions", 0) >= 1 # 这次请求自己就占着一条
assert Path(make_url(settings.database_url).database).exists()
# ---------------------------------------------------------------- 尝试次数账本
def test_reclaimed_attempt_is_refunded(api, studio, script_text, make_image_png):
"""热重载打断的那一次尝试不能算进重试额度。
attempts 记的是「开始执行了几次」,重试额度保护的是「模型侧失败几次」。
开发时反复改代码热重载会把前者刷爆:本机一个任务什么都没失败就跑到 6/3,
之后任何一个瞬时抖动都会直接把它判死。
"""
task_id = _queued_shot_task(api, studio["project_id"], script_text, make_image_png, "0x226699")
result = _make_zombie(task_id, attempts=3)
assert result["recovered"]["count"] == 1
assert result["checkpoint"]["recovered_from"] == "dead_runner"
assert result["checkpoint"]["attempt_refunded"] is True
assert result["attempts"] == 2, "被打断的那次尝试要退还"
assert result["status"] == "queued"
def test_stale_running_attempt_is_not_refunded(api, studio, script_text, make_image_png):
"""心跳超时可能真的还在别处跑着,这种不退还,免得崩坏的任务无限循环。"""
task_id = _queued_shot_task(api, studio["project_id"], script_text, make_image_png, "0x993355")
result = _make_zombie(task_id, attempts=2, dead_process=False, stale_seconds=30)
assert result["checkpoint"]["recovered_from"] == "stale_running"
assert "attempt_refunded" not in result["checkpoint"]
assert result["attempts"] == 2
def test_refund_leaves_a_working_retry_budget(api, studio, script_text, make_image_png):
"""退还不会把兜底改松:真失败照样按剩余额度重试,用尽后落败。"""
pid = studio["project_id"]
task_id = _queued_shot_task(api, pid, script_text, make_image_png, "0x117755")
spec = next(r for r in api.get("/api/providers")["providers"]
if r["provider"] == "mock" and r["media_type"] == "video")
api.patch(f"/api/providers/{spec['id']}", json={"config": {"failure_rate": 1.0}})
# 被打断在 2/3:退还后仍留 1 次 → 真失败还能重试一次,然后按额度落败
assert _make_zombie(task_id, attempts=2)["attempts"] == 1
api.drain(pid) # 断点续跑后已是 queued,直接派发
back = api.get(f"/api/tasks/{task_id}")["task"]
assert back["status"] == "failed", back
assert back["attempts"] == back["max_attempts"] == 3, back
assert [h["code"] for h in back["error_history"]] == ["mock_injected_failure"] * 2