nex_animation/backend/tests/test_startup_lock.py

129 lines
5.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""后端启动期的 SQLite 等锁逻辑(``db.wait_for_lock``)。
线上现象:改了个文件、后端热重载之后就再也连不上了——浏览器里所有请求都在转圈,
``/health`` 也不返回。日志里其实是 ``sqlite3.OperationalError: database is locked``
+ ``Application startup failed. Exiting.``:上一个 worker 还握着写事务(内联生成任务的
会话要跨过几分钟的 ComfyUI 调用),新 worker 的 ``init_db()`` 抢不到锁直接退出,
而 ``--reload`` 的父进程仍然占着端口——连得上、永不响应,看起来就是「整个平台卡住」。
"""
from __future__ import annotations
import sqlite3
import threading
import time
from pathlib import Path
import pytest
from sqlalchemy.exc import OperationalError
from app import db as dbmod
def _locked_error() -> OperationalError:
return OperationalError("UPDATE model_specs SET model_code='' ", {},
sqlite3.OperationalError("database is locked"))
def test_lock_error_is_recognised_but_real_sql_mistakes_are_not():
assert dbmod.is_lock_error(_locked_error()) is True
assert dbmod.is_lock_error(sqlite3.OperationalError("no such column: nope")) is False
assert dbmod.is_lock_error(RuntimeError("boom")) is False
def test_startup_maintenance_waits_out_a_transient_lock():
"""锁松开就立刻继续:热重载不再需要人工重启第二次。"""
calls = {"n": 0}
def run() -> str:
calls["n"] += 1
if calls["n"] < 3:
raise _locked_error()
return "done"
assert dbmod.wait_for_lock("回填新增列默认值", run, budget=10, interval=0.05) == "done"
assert calls["n"] == 3
def test_startup_maintenance_still_raises_real_errors_immediately():
"""语句本身写错时不许被「等锁」吞掉——否则一个 bug 会变成 60s 的静默停顿。"""
calls = {"n": 0}
def run() -> None:
calls["n"] += 1
raise OperationalError("SELECT 1", {}, sqlite3.OperationalError("no such table: model_specs"))
with pytest.raises(OperationalError):
dbmod.wait_for_lock("补齐新增列", run, budget=5, interval=0.05)
assert calls["n"] == 1
def test_giving_up_explains_how_to_fix_it():
with pytest.raises(RuntimeError) as caught:
dbmod.wait_for_lock("回填新增列默认值", lambda: (_ for _ in ()).throw(_locked_error()),
budget=0.2, interval=0.05)
message = str(caught.value)
assert "上一个后端进程" in message and "busy_timeout" in message and "回填新增列默认值" in message
def test_real_sqlite_write_lock_is_waited_for(tmp_path: Path):
"""真文件锁:另一个连接握着写事务时等它提交,然后维护语句正常执行。"""
path = tmp_path / "locked.db"
holder = sqlite3.connect(path, timeout=5, check_same_thread=False) # 由后台线程负责释放
holder.execute("CREATE TABLE t (x)")
holder.execute("INSERT INTO t VALUES (1)")
holder.commit()
holder.execute("BEGIN IMMEDIATE") # 独占写锁,直到 commit
released = {"at": 0.0}
def maintain() -> None:
conn = sqlite3.connect(path, timeout=0.05) # 几乎不等,把重试交给 wait_for_lock
try:
conn.execute("UPDATE t SET x = 2") # 需要写锁
conn.commit()
finally:
conn.close()
released["at"] = time.monotonic()
def release_later() -> None:
time.sleep(0.6)
holder.commit()
holder.close()
thread = threading.Thread(target=release_later, daemon=True)
thread.start()
started = time.monotonic()
try:
dbmod.wait_for_lock("真锁验收", maintain, budget=15, interval=0.1)
finally:
thread.join(timeout=5)
assert released["at"] - started >= 0.4 # 确实等过了,不是碰运气一次就过
check = sqlite3.connect(path)
try:
assert check.execute("SELECT x FROM t").fetchone()[0] == 2
finally:
check.close()
def test_init_db_survives_a_busy_database(monkeypatch):
"""把 init_db 的四步全换成「先抛两次锁」,确认启动路径整体是等锁而不是崩掉。"""
counters = {"create_all": 0, "columns": 0, "relax": 0, "backfill": 0}
def flaky(key: str):
def run(*_args, **_kwargs) -> None:
counters[key] += 1
if counters[key] < 3:
raise _locked_error()
return run
metadata = type("M", (), {"create_all": staticmethod(flaky("create_all"))})()
monkeypatch.setattr(dbmod, "Base", type("B", (), {"metadata": metadata})())
monkeypatch.setattr(dbmod, "_add_missing_columns", flaky("columns"))
# tasks.project_id 放开非空要整表重建 tasks,这一步同样在启动路径上,必须一起等锁
monkeypatch.setattr(dbmod, "_relax_nullability", flaky("relax"))
monkeypatch.setattr(dbmod, "_backfill_defaults", flaky("backfill"))
monkeypatch.setattr(dbmod, "STARTUP_LOCK_BUDGET_SECONDS", 10.0)
dbmod.init_db()
assert counters == {"create_all": 3, "columns": 3, "relax": 3, "backfill": 3}