unis-oa/backend/scripts/check_hub_sync_lock_leak.py

333 lines
14 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
"""守「拿锁之后、进 try 之前」那一段:建同步记录失败不许把锁永久扣在进程上。
踩过的坑:``sync_users`` / ``_pull_changes`` 先 :func:`app.service.hub_sync._acquire_sync_locks`
拿锁,再写一条「进行中」的记录,**这一段在收尾那个 ``try`` 之外**。库断了、死锁、
补列没跟上导致 INSERT 报错,异常直接从这段漏出去,``finally`` 够不着,进程内那把
``_run_lock`` 就永久握着了。症状是"明明没在同步,点立即同步一直回**已有同步任务正在
执行,请稍后再试**":跨副本那把 Redis 锁最多一小时自己过期,本进程这把只能等重启,
或者谁去按一下「终止」。而调度线程每几十秒那趟增量同步撞的也是同一把锁,于是整条
组织同步一起停摆。
这里钉五条::
cd backend && PYTHONPATH=. python3 scripts/check_hub_sync_lock_leak.py
1. **全量:建记录失败照样交锁**:INSERT 那一步抛错,异常原样往外走,但本进程那把锁
交还了、状态不再算运行中;
2. **增量:写游标与建记录失败照样交锁**:``_pull_changes`` 那一段同理;
3. **锁没被放松成摆设**:真跑起来的那趟在拉目录期间,第二次拿锁仍然被挡成
「已有同步任务正在执行」,跑完之后两把锁(含跨副本那把)都交还干净;
4. **紧接着再点一次进得来**:漏锁那次失败之后**不做任何复位**,全量与增量各再跑
一次,撞回来的该是 Hub 的报错而不是那句"已经有一趟在跑" —— 这一条就是修之前会
FAIL 的那一条,也正是界面上"明明没同步却提示已有同步任务"的成因;
5. **失败的那趟自己收尾**:记录写成「失败」,不留挂着「进行中」的僵尸,一个部门、
一个人都没往库里写。
不碰真 Redis(换成假锁),Hub 换成假的(``directory`` 直接报错,一个部门一个人都
不往库里写),库里只写自己造的记录,跑完按标记删干净。
"""
from __future__ import annotations
import pathlib
import sys
import threading
from datetime import datetime
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
from sqlalchemy import delete, select # noqa: E402
from app import hubconfig, models # noqa: E402
from app.db import SessionLocal # noqa: E402
from app.service import hub as hub_api # noqa: E402
from app.service import hub_sync # noqa: E402
MARK = "zz-check-hub-lock-leak"
NAME = "漏锁校核"
BUSY_MESSAGE = "已有同步任务正在执行,请稍后再试"
FAILS: list[str] = []
PASSED = 0
#: 只顶掉这几档配置:这台机器上 Hub 大概率没真启用,而这几档决定"让不让进同步"
CFG_OVERRIDE: dict[str, object] = {
"hub_enabled": True,
"hub_base_url": "https://hub.invalid",
"sync_allowed": True,
"hub_sync_enabled": True,
"hub_sync_changes_enabled": True,
"hub_sync_skip_unchanged": False,
}
class FakeCache:
"""假锁:只记"有没有人拿着"。真 Redis 上 force_release 会删掉别人正拿着的锁,
校验收脚本不该干这事(与 ``check_hub_sync_cancel.py`` 同一个理由)。"""
def __init__(self) -> None:
self.holder = ""
self.store: dict[str, object] = {}
def key(self, *parts) -> str: # noqa: ANN002
return ":".join(str(part) for part in parts)
def set_json(self, name: str, value: object, ttl_seconds: float) -> bool:
self.store[name] = value
return True
def get_json(self, name: str): # noqa: ANN201
return self.store.get(name)
def lock_holder(self, name: str) -> str:
return self.holder
def locked(self, name: str) -> bool:
return bool(self.holder)
def try_lock(self, name: str, *, ttl_seconds: float) -> str:
if self.holder:
return ""
self.holder = f"fake-{threading.get_ident()}"
return self.holder
def release_lock(self, name: str, token: str) -> None:
if token and self.holder == token:
self.holder = ""
def force_release(self, name: str) -> bool:
had = bool(self.holder)
self.holder = ""
return had
class ForcedCfg:
"""真配置照抄,只改 :data:`CFG_OVERRIDE` 里那几档。"""
def __init__(self, real: object) -> None:
self._real = real
def __getattr__(self, name: str): # noqa: ANN204
if name in CFG_OVERRIDE:
return CFG_OVERRIDE[name]
return getattr(self._real, name)
class BreakOnCommit:
"""Session 的壳子:第一次 ``commit()`` 就抛,模拟"建同步记录那一步库断了"。
只破第一次提交,正好落在 ``_acquire_sync_locks()`` 与那个收尾 ``try`` 之间;
其余调用都转给真 session,所以锁交还之后 session 还能继续用。
"""
def __init__(self, db) -> None: # noqa: ANN001
self._db = db
self.broken = False
def commit(self) -> None:
if not self.broken:
self.broken = True
raise OSError("检查脚本:写同步记录时库断了")
self._db.commit()
def __getattr__(self, name: str): # noqa: ANN204
return getattr(self._db, name)
def check(label: str, passed: bool, detail: str = "") -> None:
global PASSED
if passed:
PASSED += 1
print(f"[通过] {label}")
else:
FAILS.append(label)
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
def local_lock_free() -> bool:
"""本进程那把锁现在空不空。问完就还原:拿锁来"探测"却不还,探测本身就把锁占了。"""
if hub_sync._run_lock.acquire(blocking=False):
hub_sync._run_lock.release()
return True
return False
def reset_locks() -> None:
"""把两把锁恢复成"没人拿",让每一条校验各验各的,不被上一条的状态串。"""
if not local_lock_free():
try: # 真漏在别的线程上:跨线程 release 是允许的,正是这一段的考点
hub_sync._run_lock.release()
except RuntimeError: # pragma: no cover - 只在竞争下走到
pass
hub_sync.cache.force_release(hub_sync.LOCK_SYNC) # type: ignore[attr-defined]
def my_runs(db) -> list[models.HubSyncRun]: # noqa: ANN001
return list(
db.execute(select(models.HubSyncRun).where(models.HubSyncRun.operator == MARK)).scalars().all()
)
def call_and_catch(fn, *args, **kwargs): # noqa: ANN001, ANN002, ANN003
"""跑一趟同步,把异常压成一句话返回,校验只看这句话。"""
try:
fn(*args, **kwargs)
except Exception as exc: # noqa: BLE001 - 校验要的正就是异常本身
return exc
return None
def main() -> int:
real_cache = hub_sync.cache
real_hubconfig_get = hub_sync.hubconfig.get
real_client = hub_sync.hub_api.client
real_token = hub_sync.hub_api.sync_token
hub_sync.cache = FakeCache() # type: ignore[assignment]
hub_sync.hubconfig.get = lambda: ForcedCfg(real_hubconfig_get())
hub_sync.hub_api.sync_token = lambda cli: ("token", "directory_key")
class BrokenHub:
"""拉目录就直接失败:一个部门、一个人都不会往库里写。"""
def directory(self, token, **kwargs): # noqa: ANN001, ANN003
raise hub_api.HubError("检查脚本:Hub 挂了", reason="CHECK")
def directory_changes(self, *args, **kwargs): # noqa: ANN001, ANN003
raise hub_api.HubError("检查脚本:Hub 挂了", reason="CHECK")
blocked: dict[str, str] = {}
class BusyHub(BrokenHub):
"""同上,只是在拉目录那一下顺手再抢一次锁,验锁没被放松成摆设。"""
def directory(self, token, **kwargs): # noqa: ANN001, ANN003
blocked["message"] = second_acquire()
raise hub_api.HubError("检查脚本:Hub 挂了", reason="CHECK")
db = SessionLocal()
try:
# ① 全量:建「进行中」记录那一步炸 => 异常照旧抛,锁得还回来
reset_locks()
error = call_and_catch(
hub_sync.sync_users, db=BreakOnCommit(db), trigger=models.HubSyncRun.TRIGGER_MANUAL,
operator=MARK, operator_name=NAME,
)
check("① 全量:建记录失败时异常原样往外走", isinstance(error, OSError), repr(error))
check("① 全量:本进程那把锁已交还", local_lock_free())
check("① 全量:跨副本锁也已交还", hub_sync.cache.holder == "") # type: ignore[attr-defined]
check("① 全量:状态不再算运行中", hub_sync.running() is False)
db.rollback()
# ④ 就地复现用户那一句:漏锁之后不复位、直接再点「立即同步」。
# 修之前这里撞回来的是"已有同步任务正在执行",修之后才是 Hub 自己的报错。
hub_sync.hub_api.client = lambda: BrokenHub()
error = call_and_catch(
hub_sync.sync_users, db=db, trigger=models.HubSyncRun.TRIGGER_MANUAL,
operator=MARK, operator_name=NAME,
)
check(
"④ 全量:失败之后紧接着再点,进得来(不被误报成「已有同步任务正在执行」)",
isinstance(error, hub_sync.HubSyncError) and BUSY_MESSAGE not in str(error),
repr(error),
)
check("④ 这一趟同样把两把锁交还了", hub_sync.running() is False and hub_sync.cache.holder == "") # type: ignore[attr-defined]
# ② 增量:写游标 + 建记录那一步炸 => 同样得把锁还回来
reset_locks()
error = call_and_catch(
hub_sync.sync_changes, db=BreakOnCommit(db), operator=MARK, operator_name=NAME, manual=True,
)
check("② 增量:建记录失败时异常原样往外走", isinstance(error, OSError), repr(error))
check("② 增量:本进程那把锁已交还", local_lock_free())
check("② 增量:跨副本锁也已交还", hub_sync.cache.holder == "") # type: ignore[attr-defined]
check("② 增量:状态不再算运行中", hub_sync.running() is False)
db.rollback()
hub_sync.hub_api.client = lambda: BrokenHub()
error = call_and_catch(hub_sync.sync_changes, db=db, operator=MARK, operator_name=NAME, manual=True)
check(
"④ 增量:失败之后紧接着再拉一次,进得来",
isinstance(error, hub_sync.HubSyncError) and BUSY_MESSAGE not in str(error),
repr(error),
)
check("④ 这一趟同样把两把锁交还了", hub_sync.running() is False and hub_sync.cache.holder == "") # type: ignore[attr-defined]
# ③ 真跑起来的那一趟必须还持着锁:拉目录期间第二次拿锁要能被挡下
def second_acquire() -> str:
"""在同步中间再抢一次锁:挡下来就返回那句提示,真抢到了返回空串。"""
try:
token = hub_sync._acquire_sync_locks()
except hub_sync.HubSyncError as exc:
return str(exc)
hub_sync._release_sync_locks(token)
return ""
reset_locks()
hub_sync.hub_api.client = lambda: BusyHub()
error = call_and_catch(
hub_sync.sync_users, db=db, trigger=models.HubSyncRun.TRIGGER_MANUAL,
operator=MARK, operator_name=NAME,
)
check(
"③ 跑起来的这趟撞见的是 Hub 的报错",
isinstance(error, hub_sync.HubSyncError) and "Hub 挂了" in str(error),
repr(error),
)
check(
"③ 正在跑的时候第二次拿锁被挡成「已有同步任务正在执行」",
blocked.get("message") == BUSY_MESSAGE,
blocked.get("message", "压根没问第二次"),
)
check(
"③ 跑完之后两把锁都交还干净",
hub_sync.running() is False and local_lock_free() and hub_sync.cache.holder == "", # type: ignore[attr-defined]
)
check(
"⑤ 假 Hub 没往库里写人",
db.execute(select(models.User).where(models.User.account == MARK)).scalars().first() is None,
)
rows = my_runs(db)
check(
"⑤ 跑过的每一趟都收尾成「失败」,没留下挂着「进行中」的僵尸",
bool(rows) and all(row.status == models.HubSyncRun.STATUS_FAILED for row in rows),
" / ".join(f"{row.status}@{row.started_at}" for row in rows) or "一条都没有",
)
finally:
cleanup = SessionLocal()
try:
ids = list(cleanup.execute(select(models.HubSyncRun.id).where(
models.HubSyncRun.operator == MARK)).scalars().all())
if ids:
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(ids)))
cleanup.commit()
# 保险:漏锁那条路上如果真留下了「进行中」,一并按标记收掉,别脏了别人的页面
left = list(cleanup.execute(select(models.HubSyncRun.id).where(
models.HubSyncRun.operator_name == NAME,
models.HubSyncRun.status == models.HubSyncRun.STATUS_RUNNING)).scalars().all())
if left:
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(left)))
cleanup.commit()
print(f"[清理] 删掉 {len(ids) + len(left)} 条临时记录(时间 {datetime.now():%H:%M:%S})")
finally:
cleanup.close()
hub_sync.cache = real_cache # type: ignore[assignment]
hub_sync.hubconfig.get = real_hubconfig_get
hub_sync.hub_api.client = real_client
hub_sync.hub_api.sync_token = real_token
db.close()
print()
if FAILS:
print(f"未通过 {len(FAILS)} 项:")
for name in FAILS:
print(f" - {name}")
return 1
print(f"全部通过({PASSED} 项):建同步记录那一步出不了锁,「已有同步任务正在执行」不会自己长住。")
return 0
if __name__ == "__main__":
raise SystemExit(main())