333 lines
14 KiB
Python
333 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""守「拿锁之后、进 try 之前」那一段:建同步记录失败不许把锁永久扣在进程上。
|
||
|
||
踩过的坑:``sync_users`` / ``_pull_changes`` 先 :func:`app.service.hub_sync._acquire_sync_locks`
|
||
拿锁,再写一条「进行中」的记录,**这一段在收尾那个 ``try`` 之外**。库断了、死锁、
|
||
补列没跟上导致 INSERT 报错,异常直接从这段漏出去,``finally`` 够不着,进程内那把
|
||
``_run_lock`` 就永久握着了。症状是"明明没在同步,点立即同步一直回**已有同步任务正在
|
||
执行,请稍后再试**":跨副本那把 Redis 锁最多一小时自己过期,本进程这把只能等重启,
|
||
或者谁去按一下「终止」。而调度线程每几十秒那趟增量同步撞的也是同一把锁,于是整条
|
||
组织同步一起停摆。
|
||
|
||
这里钉五条::
|
||
|
||
cd backend && PYTHONPATH=. python3 scripts/check_hub_sync_lock_leak.py
|
||
|
||
1. **全量:建记录失败照样交锁**:INSERT 那一步抛错,异常原样往外走,但本进程那把锁
|
||
交还了、状态不再算运行中;
|
||
2. **增量:写游标与建记录失败照样交锁**:``_pull_changes`` 那一段同理;
|
||
3. **锁没被放松成摆设**:真跑起来的那趟在拉目录期间,第二次拿锁仍然被挡成
|
||
「已有同步任务正在执行」,跑完之后两把锁(含跨副本那把)都交还干净;
|
||
4. **紧接着再点一次进得来**:漏锁那次失败之后**不做任何复位**,全量与增量各再跑
|
||
一次,撞回来的该是 Hub 的报错而不是那句"已经有一趟在跑" —— 这一条就是修之前会
|
||
FAIL 的那一条,也正是界面上"明明没同步却提示已有同步任务"的成因;
|
||
5. **失败的那趟自己收尾**:记录写成「失败」,不留挂着「进行中」的僵尸,一个部门、
|
||
一个人都没往库里写。
|
||
|
||
不碰真 Redis(换成假锁),Hub 换成假的(``directory`` 直接报错,一个部门一个人都
|
||
不往库里写),库里只写自己造的记录,跑完按标记删干净。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pathlib
|
||
import sys
|
||
import threading
|
||
from datetime import datetime
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
||
|
||
from sqlalchemy import delete, select # noqa: E402
|
||
|
||
from app import hubconfig, models # noqa: E402
|
||
from app.db import SessionLocal # noqa: E402
|
||
from app.service import hub as hub_api # noqa: E402
|
||
from app.service import hub_sync # noqa: E402
|
||
|
||
MARK = "zz-check-hub-lock-leak"
|
||
NAME = "漏锁校核"
|
||
BUSY_MESSAGE = "已有同步任务正在执行,请稍后再试"
|
||
FAILS: list[str] = []
|
||
PASSED = 0
|
||
|
||
#: 只顶掉这几档配置:这台机器上 Hub 大概率没真启用,而这几档决定"让不让进同步"
|
||
CFG_OVERRIDE: dict[str, object] = {
|
||
"hub_enabled": True,
|
||
"hub_base_url": "https://hub.invalid",
|
||
"sync_allowed": True,
|
||
"hub_sync_enabled": True,
|
||
"hub_sync_changes_enabled": True,
|
||
"hub_sync_skip_unchanged": False,
|
||
}
|
||
|
||
|
||
class FakeCache:
|
||
"""假锁:只记"有没有人拿着"。真 Redis 上 force_release 会删掉别人正拿着的锁,
|
||
校验收脚本不该干这事(与 ``check_hub_sync_cancel.py`` 同一个理由)。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.holder = ""
|
||
self.store: dict[str, object] = {}
|
||
|
||
def key(self, *parts) -> str: # noqa: ANN002
|
||
return ":".join(str(part) for part in parts)
|
||
|
||
def set_json(self, name: str, value: object, ttl_seconds: float) -> bool:
|
||
self.store[name] = value
|
||
return True
|
||
|
||
def get_json(self, name: str): # noqa: ANN201
|
||
return self.store.get(name)
|
||
|
||
def lock_holder(self, name: str) -> str:
|
||
return self.holder
|
||
|
||
def locked(self, name: str) -> bool:
|
||
return bool(self.holder)
|
||
|
||
def try_lock(self, name: str, *, ttl_seconds: float) -> str:
|
||
if self.holder:
|
||
return ""
|
||
self.holder = f"fake-{threading.get_ident()}"
|
||
return self.holder
|
||
|
||
def release_lock(self, name: str, token: str) -> None:
|
||
if token and self.holder == token:
|
||
self.holder = ""
|
||
|
||
def force_release(self, name: str) -> bool:
|
||
had = bool(self.holder)
|
||
self.holder = ""
|
||
return had
|
||
|
||
|
||
class ForcedCfg:
|
||
"""真配置照抄,只改 :data:`CFG_OVERRIDE` 里那几档。"""
|
||
|
||
def __init__(self, real: object) -> None:
|
||
self._real = real
|
||
|
||
def __getattr__(self, name: str): # noqa: ANN204
|
||
if name in CFG_OVERRIDE:
|
||
return CFG_OVERRIDE[name]
|
||
return getattr(self._real, name)
|
||
|
||
|
||
class BreakOnCommit:
|
||
"""Session 的壳子:第一次 ``commit()`` 就抛,模拟"建同步记录那一步库断了"。
|
||
|
||
只破第一次提交,正好落在 ``_acquire_sync_locks()`` 与那个收尾 ``try`` 之间;
|
||
其余调用都转给真 session,所以锁交还之后 session 还能继续用。
|
||
"""
|
||
|
||
def __init__(self, db) -> None: # noqa: ANN001
|
||
self._db = db
|
||
self.broken = False
|
||
|
||
def commit(self) -> None:
|
||
if not self.broken:
|
||
self.broken = True
|
||
raise OSError("检查脚本:写同步记录时库断了")
|
||
self._db.commit()
|
||
|
||
def __getattr__(self, name: str): # noqa: ANN204
|
||
return getattr(self._db, name)
|
||
|
||
|
||
def check(label: str, passed: bool, detail: str = "") -> None:
|
||
global PASSED
|
||
if passed:
|
||
PASSED += 1
|
||
print(f"[通过] {label}")
|
||
else:
|
||
FAILS.append(label)
|
||
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
|
||
|
||
|
||
def local_lock_free() -> bool:
|
||
"""本进程那把锁现在空不空。问完就还原:拿锁来"探测"却不还,探测本身就把锁占了。"""
|
||
if hub_sync._run_lock.acquire(blocking=False):
|
||
hub_sync._run_lock.release()
|
||
return True
|
||
return False
|
||
|
||
|
||
def reset_locks() -> None:
|
||
"""把两把锁恢复成"没人拿",让每一条校验各验各的,不被上一条的状态串。"""
|
||
if not local_lock_free():
|
||
try: # 真漏在别的线程上:跨线程 release 是允许的,正是这一段的考点
|
||
hub_sync._run_lock.release()
|
||
except RuntimeError: # pragma: no cover - 只在竞争下走到
|
||
pass
|
||
hub_sync.cache.force_release(hub_sync.LOCK_SYNC) # type: ignore[attr-defined]
|
||
|
||
|
||
def my_runs(db) -> list[models.HubSyncRun]: # noqa: ANN001
|
||
return list(
|
||
db.execute(select(models.HubSyncRun).where(models.HubSyncRun.operator == MARK)).scalars().all()
|
||
)
|
||
|
||
|
||
def call_and_catch(fn, *args, **kwargs): # noqa: ANN001, ANN002, ANN003
|
||
"""跑一趟同步,把异常压成一句话返回,校验只看这句话。"""
|
||
try:
|
||
fn(*args, **kwargs)
|
||
except Exception as exc: # noqa: BLE001 - 校验要的正就是异常本身
|
||
return exc
|
||
return None
|
||
|
||
|
||
def main() -> int:
|
||
real_cache = hub_sync.cache
|
||
real_hubconfig_get = hub_sync.hubconfig.get
|
||
real_client = hub_sync.hub_api.client
|
||
real_token = hub_sync.hub_api.sync_token
|
||
|
||
hub_sync.cache = FakeCache() # type: ignore[assignment]
|
||
hub_sync.hubconfig.get = lambda: ForcedCfg(real_hubconfig_get())
|
||
hub_sync.hub_api.sync_token = lambda cli: ("token", "directory_key")
|
||
|
||
class BrokenHub:
|
||
"""拉目录就直接失败:一个部门、一个人都不会往库里写。"""
|
||
|
||
def directory(self, token, **kwargs): # noqa: ANN001, ANN003
|
||
raise hub_api.HubError("检查脚本:Hub 挂了", reason="CHECK")
|
||
|
||
def directory_changes(self, *args, **kwargs): # noqa: ANN001, ANN003
|
||
raise hub_api.HubError("检查脚本:Hub 挂了", reason="CHECK")
|
||
|
||
blocked: dict[str, str] = {}
|
||
|
||
class BusyHub(BrokenHub):
|
||
"""同上,只是在拉目录那一下顺手再抢一次锁,验锁没被放松成摆设。"""
|
||
|
||
def directory(self, token, **kwargs): # noqa: ANN001, ANN003
|
||
blocked["message"] = second_acquire()
|
||
raise hub_api.HubError("检查脚本:Hub 挂了", reason="CHECK")
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
# ① 全量:建「进行中」记录那一步炸 => 异常照旧抛,锁得还回来
|
||
reset_locks()
|
||
error = call_and_catch(
|
||
hub_sync.sync_users, db=BreakOnCommit(db), trigger=models.HubSyncRun.TRIGGER_MANUAL,
|
||
operator=MARK, operator_name=NAME,
|
||
)
|
||
check("① 全量:建记录失败时异常原样往外走", isinstance(error, OSError), repr(error))
|
||
check("① 全量:本进程那把锁已交还", local_lock_free())
|
||
check("① 全量:跨副本锁也已交还", hub_sync.cache.holder == "") # type: ignore[attr-defined]
|
||
check("① 全量:状态不再算运行中", hub_sync.running() is False)
|
||
db.rollback()
|
||
|
||
# ④ 就地复现用户那一句:漏锁之后不复位、直接再点「立即同步」。
|
||
# 修之前这里撞回来的是"已有同步任务正在执行",修之后才是 Hub 自己的报错。
|
||
hub_sync.hub_api.client = lambda: BrokenHub()
|
||
error = call_and_catch(
|
||
hub_sync.sync_users, db=db, trigger=models.HubSyncRun.TRIGGER_MANUAL,
|
||
operator=MARK, operator_name=NAME,
|
||
)
|
||
check(
|
||
"④ 全量:失败之后紧接着再点,进得来(不被误报成「已有同步任务正在执行」)",
|
||
isinstance(error, hub_sync.HubSyncError) and BUSY_MESSAGE not in str(error),
|
||
repr(error),
|
||
)
|
||
check("④ 这一趟同样把两把锁交还了", hub_sync.running() is False and hub_sync.cache.holder == "") # type: ignore[attr-defined]
|
||
|
||
# ② 增量:写游标 + 建记录那一步炸 => 同样得把锁还回来
|
||
reset_locks()
|
||
error = call_and_catch(
|
||
hub_sync.sync_changes, db=BreakOnCommit(db), operator=MARK, operator_name=NAME, manual=True,
|
||
)
|
||
check("② 增量:建记录失败时异常原样往外走", isinstance(error, OSError), repr(error))
|
||
check("② 增量:本进程那把锁已交还", local_lock_free())
|
||
check("② 增量:跨副本锁也已交还", hub_sync.cache.holder == "") # type: ignore[attr-defined]
|
||
check("② 增量:状态不再算运行中", hub_sync.running() is False)
|
||
db.rollback()
|
||
|
||
hub_sync.hub_api.client = lambda: BrokenHub()
|
||
error = call_and_catch(hub_sync.sync_changes, db=db, operator=MARK, operator_name=NAME, manual=True)
|
||
check(
|
||
"④ 增量:失败之后紧接着再拉一次,进得来",
|
||
isinstance(error, hub_sync.HubSyncError) and BUSY_MESSAGE not in str(error),
|
||
repr(error),
|
||
)
|
||
check("④ 这一趟同样把两把锁交还了", hub_sync.running() is False and hub_sync.cache.holder == "") # type: ignore[attr-defined]
|
||
|
||
# ③ 真跑起来的那一趟必须还持着锁:拉目录期间第二次拿锁要能被挡下
|
||
def second_acquire() -> str:
|
||
"""在同步中间再抢一次锁:挡下来就返回那句提示,真抢到了返回空串。"""
|
||
try:
|
||
token = hub_sync._acquire_sync_locks()
|
||
except hub_sync.HubSyncError as exc:
|
||
return str(exc)
|
||
hub_sync._release_sync_locks(token)
|
||
return ""
|
||
|
||
reset_locks()
|
||
hub_sync.hub_api.client = lambda: BusyHub()
|
||
error = call_and_catch(
|
||
hub_sync.sync_users, db=db, trigger=models.HubSyncRun.TRIGGER_MANUAL,
|
||
operator=MARK, operator_name=NAME,
|
||
)
|
||
check(
|
||
"③ 跑起来的这趟撞见的是 Hub 的报错",
|
||
isinstance(error, hub_sync.HubSyncError) and "Hub 挂了" in str(error),
|
||
repr(error),
|
||
)
|
||
check(
|
||
"③ 正在跑的时候第二次拿锁被挡成「已有同步任务正在执行」",
|
||
blocked.get("message") == BUSY_MESSAGE,
|
||
blocked.get("message", "压根没问第二次"),
|
||
)
|
||
check(
|
||
"③ 跑完之后两把锁都交还干净",
|
||
hub_sync.running() is False and local_lock_free() and hub_sync.cache.holder == "", # type: ignore[attr-defined]
|
||
)
|
||
check(
|
||
"⑤ 假 Hub 没往库里写人",
|
||
db.execute(select(models.User).where(models.User.account == MARK)).scalars().first() is None,
|
||
)
|
||
|
||
rows = my_runs(db)
|
||
check(
|
||
"⑤ 跑过的每一趟都收尾成「失败」,没留下挂着「进行中」的僵尸",
|
||
bool(rows) and all(row.status == models.HubSyncRun.STATUS_FAILED for row in rows),
|
||
" / ".join(f"{row.status}@{row.started_at}" for row in rows) or "一条都没有",
|
||
)
|
||
finally:
|
||
cleanup = SessionLocal()
|
||
try:
|
||
ids = list(cleanup.execute(select(models.HubSyncRun.id).where(
|
||
models.HubSyncRun.operator == MARK)).scalars().all())
|
||
if ids:
|
||
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(ids)))
|
||
cleanup.commit()
|
||
# 保险:漏锁那条路上如果真留下了「进行中」,一并按标记收掉,别脏了别人的页面
|
||
left = list(cleanup.execute(select(models.HubSyncRun.id).where(
|
||
models.HubSyncRun.operator_name == NAME,
|
||
models.HubSyncRun.status == models.HubSyncRun.STATUS_RUNNING)).scalars().all())
|
||
if left:
|
||
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(left)))
|
||
cleanup.commit()
|
||
print(f"[清理] 删掉 {len(ids) + len(left)} 条临时记录(时间 {datetime.now():%H:%M:%S})")
|
||
finally:
|
||
cleanup.close()
|
||
hub_sync.cache = real_cache # type: ignore[assignment]
|
||
hub_sync.hubconfig.get = real_hubconfig_get
|
||
hub_sync.hub_api.client = real_client
|
||
hub_sync.hub_api.sync_token = real_token
|
||
db.close()
|
||
|
||
print()
|
||
if FAILS:
|
||
print(f"未通过 {len(FAILS)} 项:")
|
||
for name in FAILS:
|
||
print(f" - {name}")
|
||
return 1
|
||
print(f"全部通过({PASSED} 项):建同步记录那一步出不了锁,「已有同步任务正在执行」不会自己长住。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|