170 lines
7.6 KiB
Python
170 lines
7.6 KiB
Python
"""数据库连接池:长命会话不得把别人挡在池口外。
|
||
|
||
真实事故:本项目用的是文件 SQLite,SQLAlchemy 默认 QueuePool 只有 5 常驻 + 10 溢出。
|
||
而内联 worker 一条会话要跨过几分钟的供应商调用、LLM 总控台一条会话能握几个小时,
|
||
15 条连接被这类长会话占满之后,后来者在池口排队 30 秒抛
|
||
``QueuePool limit of size 5 overflow 10 reached``,
|
||
`handle_generate_shot` 把它当成生成失败,连烧三次重试——两条已经排到 GPU 前的
|
||
视频就这么被判了死刑,而任务日志里连「数据库」三个字都没有。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
from datetime import timedelta
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.engine.url import make_url
|
||
|
||
from app.config import settings
|
||
from app.db import (_build_engine, _pool_kwargs, open_sessions, pool_status, SessionLocal,
|
||
session_trace_enabled)
|
||
|
||
|
||
def _queued_shot_task(api, pid, script_text, make_image_png, colour):
|
||
api.post(f"/api/projects/{pid}/import",
|
||
data={"script_text": script_text, "default_duration": 2},
|
||
files=[("images", ("s01.png", make_image_png(480, 270, colour), "image/png"))])
|
||
api.post(f"/api/projects/{pid}/tasks", json={"with_voiceover": False})
|
||
return api.get("/api/tasks", params={"project_id": pid, "kind": "generate_shot"})["tasks"][0]["id"]
|
||
|
||
|
||
def _make_zombie(task_id, *, attempts, dead_process=True, stale_seconds=0):
|
||
"""把任务伪装成「上一代进程遗留的 running」(可选:进程已消失 / 仅心跳超时)。"""
|
||
import socket
|
||
|
||
from app.models import Task, now_utc
|
||
|
||
if dead_process:
|
||
child = subprocess.Popen([sys.executable or "/bin/true", "-c", "pass"])
|
||
child.wait() # 一个确实已经消失的 pid
|
||
worker_id = f"{socket.gethostname()}:{child.pid}"
|
||
else:
|
||
worker_id = "inline-worker-without-a-pid" # 解析不出 pid → 只能靠心跳超时兜底
|
||
session = SessionLocal()
|
||
try:
|
||
task = session.get(Task, task_id)
|
||
task.status, task.attempts, task.max_attempts = "running", attempts, 3
|
||
task.execution_backend, task.worker_id = "inline", worker_id
|
||
task.started_at = now_utc()
|
||
task.heartbeat_at = now_utc() - timedelta(seconds=stale_seconds)
|
||
session.commit()
|
||
from app.services import dispatch
|
||
|
||
recovered = dispatch.resume_interrupted(session)
|
||
row = session.get(Task, task_id)
|
||
return {"recovered": recovered, "status": row.status, "attempts": row.attempts,
|
||
"checkpoint": dict(row.checkpoint_json or {})}
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
# ---------------------------------------------------------------- 连接池选型
|
||
def test_file_sqlite_uses_an_unbounded_pool(tmp_path):
|
||
"""文件库不再走有上限的 QueuePool;内存库仍必须共享同一条连接。"""
|
||
assert _pool_kwargs(f"sqlite:///{tmp_path}/a.db") == {"poolclass": type(_build_engine(f"sqlite:///{tmp_path}/a.db").pool)}
|
||
assert _pool_kwargs("sqlite:///:memory:")["pool_pre_ping"] is True
|
||
|
||
|
||
def test_postgres_keeps_a_real_pool():
|
||
"""Postgres 建连是 TCP+认证,必须继续池化——别把这条一起改掉。"""
|
||
kwargs = _pool_kwargs("postgresql+psycopg://u:p@localhost:5432/db")
|
||
assert kwargs["pool_size"] == 10 and kwargs["max_overflow"] == 20 and kwargs["pool_pre_ping"] is True
|
||
|
||
|
||
def test_sessions_beyond_the_old_5_plus_10_limit_coexist():
|
||
"""同时借出比旧上限(15)更多的会话:不排队、不超时。
|
||
|
||
自检库同样是文件 SQLite,所以这里直接借真引擎。旧实现下第 16 条要等满
|
||
pool_timeout(30 秒)才抛异常,这条用例会以「等待过久 + 异常」收场。
|
||
"""
|
||
held = []
|
||
started = time.monotonic()
|
||
try:
|
||
for _ in range(25):
|
||
session = SessionLocal()
|
||
held.append(session)
|
||
session.execute(text("select 1"))
|
||
assert len(held) == 25
|
||
finally:
|
||
for session in held:
|
||
session.close()
|
||
assert time.monotonic() - started < 10, "借会话不该排队等池子"
|
||
|
||
|
||
def test_writes_still_serialise_correctly():
|
||
"""NullPool 只是不复用连接,写事务的可见性语义必须原样保留。"""
|
||
from app.models import Project
|
||
|
||
name = "池子改造后的项目"
|
||
session = SessionLocal()
|
||
try:
|
||
session.add(Project(name=name))
|
||
session.commit()
|
||
finally:
|
||
session.close()
|
||
reader = SessionLocal()
|
||
try:
|
||
assert reader.query(Project).filter(Project.name == name).count() == 1
|
||
finally:
|
||
reader.close()
|
||
|
||
|
||
def test_pool_status_is_exposed_on_health(api):
|
||
"""/health 要能直接回答「是不是又在池口排队」。"""
|
||
pool = api.get("/health")["database_pool"]
|
||
assert pool["class"] in {"NullPool", "QueuePool", "StaticPool"}
|
||
if session_trace_enabled():
|
||
assert isinstance(open_sessions(), list)
|
||
assert pool.get("open_sessions", 0) >= 1 # 这次请求自己就占着一条
|
||
assert Path(make_url(settings.database_url).database).exists()
|
||
|
||
|
||
# ---------------------------------------------------------------- 尝试次数账本
|
||
def test_reclaimed_attempt_is_refunded(api, studio, script_text, make_image_png):
|
||
"""热重载打断的那一次尝试不能算进重试额度。
|
||
|
||
attempts 记的是「开始执行了几次」,重试额度保护的是「模型侧失败几次」。
|
||
开发时反复改代码热重载会把前者刷爆:本机一个任务什么都没失败就跑到 6/3,
|
||
之后任何一个瞬时抖动都会直接把它判死。
|
||
"""
|
||
task_id = _queued_shot_task(api, studio["project_id"], script_text, make_image_png, "0x226699")
|
||
result = _make_zombie(task_id, attempts=3)
|
||
|
||
assert result["recovered"]["count"] == 1
|
||
assert result["checkpoint"]["recovered_from"] == "dead_runner"
|
||
assert result["checkpoint"]["attempt_refunded"] is True
|
||
assert result["attempts"] == 2, "被打断的那次尝试要退还"
|
||
assert result["status"] == "queued"
|
||
|
||
|
||
def test_stale_running_attempt_is_not_refunded(api, studio, script_text, make_image_png):
|
||
"""心跳超时可能真的还在别处跑着,这种不退还,免得崩坏的任务无限循环。"""
|
||
task_id = _queued_shot_task(api, studio["project_id"], script_text, make_image_png, "0x993355")
|
||
result = _make_zombie(task_id, attempts=2, dead_process=False, stale_seconds=30)
|
||
|
||
assert result["checkpoint"]["recovered_from"] == "stale_running"
|
||
assert "attempt_refunded" not in result["checkpoint"]
|
||
assert result["attempts"] == 2
|
||
|
||
|
||
def test_refund_leaves_a_working_retry_budget(api, studio, script_text, make_image_png):
|
||
"""退还不会把兜底改松:真失败照样按剩余额度重试,用尽后落败。"""
|
||
pid = studio["project_id"]
|
||
task_id = _queued_shot_task(api, pid, script_text, make_image_png, "0x117755")
|
||
spec = next(r for r in api.get("/api/providers")["providers"]
|
||
if r["provider"] == "mock" and r["media_type"] == "video")
|
||
api.patch(f"/api/providers/{spec['id']}", json={"config": {"failure_rate": 1.0}})
|
||
# 被打断在 2/3:退还后仍留 1 次 → 真失败还能重试一次,然后按额度落败
|
||
assert _make_zombie(task_id, attempts=2)["attempts"] == 1
|
||
|
||
api.drain(pid) # 断点续跑后已是 queued,直接派发
|
||
back = api.get(f"/api/tasks/{task_id}")["task"]
|
||
assert back["status"] == "failed", back
|
||
assert back["attempts"] == back["max_attempts"] == 3, back
|
||
assert [h["code"] for h in back["error_history"]] == ["mock_injected_failure"] * 2
|