127 lines
4.8 KiB
Python
127 lines
4.8 KiB
Python
"""后端启动期的 SQLite 等锁逻辑(``db.wait_for_lock``)。
|
|
|
|
线上现象:改了个文件、后端热重载之后就再也连不上了——浏览器里所有请求都在转圈,
|
|
``/health`` 也不返回。日志里其实是 ``sqlite3.OperationalError: database is locked``
|
|
+ ``Application startup failed. Exiting.``:上一个 worker 还握着写事务(内联生成任务的
|
|
会话要跨过几分钟的 ComfyUI 调用),新 worker 的 ``init_db()`` 抢不到锁直接退出,
|
|
而 ``--reload`` 的父进程仍然占着端口——连得上、永不响应,看起来就是「整个平台卡住」。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy.exc import OperationalError
|
|
|
|
from app import db as dbmod
|
|
|
|
|
|
def _locked_error() -> OperationalError:
|
|
return OperationalError("UPDATE model_specs SET model_code='' ", {},
|
|
sqlite3.OperationalError("database is locked"))
|
|
|
|
|
|
def test_lock_error_is_recognised_but_real_sql_mistakes_are_not():
|
|
assert dbmod.is_lock_error(_locked_error()) is True
|
|
assert dbmod.is_lock_error(sqlite3.OperationalError("no such column: nope")) is False
|
|
assert dbmod.is_lock_error(RuntimeError("boom")) is False
|
|
|
|
|
|
def test_startup_maintenance_waits_out_a_transient_lock():
|
|
"""锁松开就立刻继续:热重载不再需要人工重启第二次。"""
|
|
calls = {"n": 0}
|
|
|
|
def run() -> str:
|
|
calls["n"] += 1
|
|
if calls["n"] < 3:
|
|
raise _locked_error()
|
|
return "done"
|
|
|
|
assert dbmod.wait_for_lock("回填新增列默认值", run, budget=10, interval=0.05) == "done"
|
|
assert calls["n"] == 3
|
|
|
|
|
|
def test_startup_maintenance_still_raises_real_errors_immediately():
|
|
"""语句本身写错时不许被「等锁」吞掉——否则一个 bug 会变成 60s 的静默停顿。"""
|
|
calls = {"n": 0}
|
|
|
|
def run() -> None:
|
|
calls["n"] += 1
|
|
raise OperationalError("SELECT 1", {}, sqlite3.OperationalError("no such table: model_specs"))
|
|
|
|
with pytest.raises(OperationalError):
|
|
dbmod.wait_for_lock("补齐新增列", run, budget=5, interval=0.05)
|
|
assert calls["n"] == 1
|
|
|
|
|
|
def test_giving_up_explains_how_to_fix_it():
|
|
with pytest.raises(RuntimeError) as caught:
|
|
dbmod.wait_for_lock("回填新增列默认值", lambda: (_ for _ in ()).throw(_locked_error()),
|
|
budget=0.2, interval=0.05)
|
|
message = str(caught.value)
|
|
assert "上一个后端进程" in message and "busy_timeout" in message and "回填新增列默认值" in message
|
|
|
|
|
|
def test_real_sqlite_write_lock_is_waited_for(tmp_path: Path):
|
|
"""真文件锁:另一个连接握着写事务时等它提交,然后维护语句正常执行。"""
|
|
path = tmp_path / "locked.db"
|
|
holder = sqlite3.connect(path, timeout=5, check_same_thread=False) # 由后台线程负责释放
|
|
holder.execute("CREATE TABLE t (x)")
|
|
holder.execute("INSERT INTO t VALUES (1)")
|
|
holder.commit()
|
|
holder.execute("BEGIN IMMEDIATE") # 独占写锁,直到 commit
|
|
|
|
released = {"at": 0.0}
|
|
|
|
def maintain() -> None:
|
|
conn = sqlite3.connect(path, timeout=0.05) # 几乎不等,把重试交给 wait_for_lock
|
|
try:
|
|
conn.execute("UPDATE t SET x = 2") # 需要写锁
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
released["at"] = time.monotonic()
|
|
|
|
def release_later() -> None:
|
|
time.sleep(0.6)
|
|
holder.commit()
|
|
holder.close()
|
|
|
|
thread = threading.Thread(target=release_later, daemon=True)
|
|
thread.start()
|
|
started = time.monotonic()
|
|
try:
|
|
dbmod.wait_for_lock("真锁验收", maintain, budget=15, interval=0.1)
|
|
finally:
|
|
thread.join(timeout=5)
|
|
assert released["at"] - started >= 0.4 # 确实等过了,不是碰运气一次就过
|
|
check = sqlite3.connect(path)
|
|
try:
|
|
assert check.execute("SELECT x FROM t").fetchone()[0] == 2
|
|
finally:
|
|
check.close()
|
|
|
|
|
|
def test_init_db_survives_a_busy_database(monkeypatch):
|
|
"""把 init_db 的三步全换成「先抛两次锁」,确认启动路径整体是等锁而不是崩掉。"""
|
|
counters = {"create_all": 0, "columns": 0, "backfill": 0}
|
|
|
|
def flaky(key: str):
|
|
def run(*_args, **_kwargs) -> None:
|
|
counters[key] += 1
|
|
if counters[key] < 3:
|
|
raise _locked_error()
|
|
return run
|
|
|
|
metadata = type("M", (), {"create_all": staticmethod(flaky("create_all"))})()
|
|
monkeypatch.setattr(dbmod, "Base", type("B", (), {"metadata": metadata})())
|
|
monkeypatch.setattr(dbmod, "_add_missing_columns", flaky("columns"))
|
|
monkeypatch.setattr(dbmod, "_backfill_defaults", flaky("backfill"))
|
|
monkeypatch.setattr(dbmod, "STARTUP_LOCK_BUDGET_SECONDS", 10.0)
|
|
|
|
dbmod.init_db()
|
|
assert counters == {"create_all": 3, "columns": 3, "backfill": 3}
|