366 lines
17 KiB
Python
366 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""守「终止同步」这条路:卡住的「进行中」必须出得去,真在跑的不许被写坏。
|
||
|
||
踩过的坑:一趟同步刚开始时后端被 kill(重启、发布、容器驱逐),
|
||
``oa_hub_sync_run`` 里那条 ``status='running'`` 就再也没人收尾 —— 同步历史一直显示
|
||
「进行中」,页头「立即同步」的 ``busy`` 也跟着永远为真、按钮按不动,只能连库手改。
|
||
这一层给两条出路:启动 / 读状态时自动收尾残留
|
||
(:func:`app.service.hub_sync.reap_stale_runs`),以及界面上那个人按的「终止」
|
||
(:func:`app.service.hub_sync.terminate_sync` + ``POST /api/v1/hub/sync/cancel``)。
|
||
|
||
这里钉七条::
|
||
|
||
cd backend && PYTHONPATH=. python3 scripts/check_hub_sync_cancel.py
|
||
|
||
1. **没到点的不算残留**:刚起步一分钟的「进行中」不动它,否则一趟正常的慢同步会被
|
||
谁读一次状态就写坏;
|
||
2. **到点又没人持锁才收尾**:超过宽限期且没有同步锁的,标成「已终止」、补结束时间与耗时;
|
||
3. **持着锁就一个字都不改**:真在跑的那趟不许被状态接口写坏;
|
||
4. **「终止」点名要动就动**:``grace_seconds=0`` 连刚起步的记录也收尾,并把锁交还;
|
||
跑在另一条线程上的那一趟随后自己 release 也不许炸;
|
||
5. **已经结束的永远不碰**:success / failed / skipped 改一条都是伪造历史;
|
||
6. **收尾不吞真结果**:被「终止」的那趟如果其实还活着,自己收尾时要把状态改回真实结果;
|
||
7. **读一次状态就自愈**:状态接口顺带收残留,页面不必等后端重启;
|
||
8. **接口那条路走得通**:``POST /api/v1/hub/sync/cancel`` 挂着 ``org:person``,没登录回
|
||
401,登录后真把库里那条「进行中」收尾,并留一条操作日志(``action=hub_sync``);
|
||
9. **旧标记不许停新同步**:终止信号按"这一趟开始之后有没有人按过"算,刚终止完立刻
|
||
重开的那一趟不能被上一趟的标记停掉;
|
||
10. **真跑起来的那一趟会被叫停**:拉目录期间按下终止,整趟停在检查点上,记录写
|
||
「已终止」而不是「失败」,一个部门、一个人都没往库里写。
|
||
|
||
不碰真 Redis(换成假锁,免得 force_release 删掉线上正拿着的那把),库里只写自己造的
|
||
临时行,跑完按标记删干净。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pathlib
|
||
import sys
|
||
import threading
|
||
import time
|
||
from datetime import datetime, timedelta
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
||
|
||
from sqlalchemy import delete, select # noqa: E402
|
||
from starlette.testclient import TestClient # noqa: E402
|
||
|
||
from app import models # noqa: E402
|
||
from app.db import SessionLocal # noqa: E402
|
||
from app.main import app # noqa: E402
|
||
from app.security import create_access_token # noqa: E402
|
||
from app.service import hub_sync # noqa: E402
|
||
|
||
MARK = "zz-check-hub-cancel"
|
||
FAILS: list[str] = []
|
||
PASSED = 0
|
||
|
||
|
||
class FakeCache:
|
||
"""假锁 + 假取值层:只记"有没有人拿着"和几个键。真 Redis 上 force_release 会删掉
|
||
别人正拿着的锁,终止标记也会写到别的副本脸上,校验收脚本不该干这事。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.holder = ""
|
||
self.store: dict[str, object] = {}
|
||
|
||
def key(self, *parts) -> str: # noqa: ANN002
|
||
return ":".join(str(part) for part in parts)
|
||
|
||
def set_json(self, name: str, value: object, ttl_seconds: float) -> bool:
|
||
self.store[name] = value
|
||
return True
|
||
|
||
def get_json(self, name: str): # noqa: ANN201
|
||
return self.store.get(name)
|
||
|
||
def delete(self, *names) -> int: # noqa: ANN002
|
||
gone = 0
|
||
for name in names:
|
||
if name in self.store:
|
||
self.store.pop(name)
|
||
gone += 1
|
||
return gone
|
||
|
||
def lock_holder(self, name: str) -> str:
|
||
return self.holder
|
||
|
||
def locked(self, name: str) -> bool:
|
||
return bool(self.holder)
|
||
|
||
def try_lock(self, name: str, *, ttl_seconds: float) -> str:
|
||
if self.holder:
|
||
return ""
|
||
self.holder = f"fake-{threading.get_ident()}"
|
||
return self.holder
|
||
|
||
def release_lock(self, name: str, token: str) -> None:
|
||
if token and self.holder == token:
|
||
self.holder = ""
|
||
|
||
def force_release(self, name: str) -> bool:
|
||
had = bool(self.holder)
|
||
self.holder = ""
|
||
return had
|
||
|
||
|
||
def check(label: str, passed: bool, detail: str = "") -> None:
|
||
global PASSED
|
||
if passed:
|
||
PASSED += 1
|
||
print(f"[通过] {label}")
|
||
else:
|
||
FAILS.append(label)
|
||
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
|
||
|
||
|
||
def add_run(db, status: str, started_at: datetime, *, minutes_ago: float = 0): # noqa: ANN001
|
||
row = models.HubSyncRun(
|
||
trigger=models.HubSyncRun.TRIGGER_CHANGES,
|
||
status=status,
|
||
started_at=started_at,
|
||
operator=MARK,
|
||
)
|
||
db.add(row)
|
||
db.commit()
|
||
return row.id
|
||
|
||
|
||
def run_of(client, headers, run_id: str) -> dict: # noqa: ANN001
|
||
"""从 ``GET /hub/sync/runs`` 里取自己造的那一条(``last_run`` 未必是它)。"""
|
||
payload = client.get("/api/v1/hub/sync/runs", params={"limit": 50}, headers=headers).json()
|
||
for row in (payload or {}).get("data") or []:
|
||
if row.get("id") == run_id:
|
||
return row
|
||
return {}
|
||
|
||
|
||
def main() -> int:
|
||
real_cache = hub_sync.cache
|
||
hub_sync.cache = FakeCache() # type: ignore[assignment]
|
||
db = SessionLocal()
|
||
try:
|
||
now = datetime.now()
|
||
|
||
# ① 没到点的不算残留
|
||
fresh = add_run(db, models.HubSyncRun.STATUS_RUNNING, now - timedelta(minutes=1))
|
||
# 这库是共享的,别的环境留下的残留会被这一次顺带收掉,所以只验自己那条没动
|
||
hub_sync.reap_stale_runs(db)
|
||
check(
|
||
"① 刚起步的「进行中」不被自动收尾",
|
||
db.get(models.HubSyncRun, fresh).status == models.HubSyncRun.STATUS_RUNNING,
|
||
)
|
||
|
||
# ② 到点且没人持锁 => 收尾;⑤ 已经结束的不动
|
||
old = add_run(db, models.HubSyncRun.STATUS_RUNNING, now - timedelta(hours=3))
|
||
done = add_run(db, models.HubSyncRun.STATUS_SUCCESS, now - timedelta(hours=4))
|
||
row = db.get(models.HubSyncRun, done)
|
||
row.finished_at = now - timedelta(hours=4, minutes=2)
|
||
db.commit()
|
||
check("② 到点的残留被收尾", hub_sync.reap_stale_runs(db, message="检查脚本收尾") == 1)
|
||
row = db.get(models.HubSyncRun, old)
|
||
check(
|
||
"② 收尾补齐状态/结束时间/耗时",
|
||
row.status == models.HubSyncRun.STATUS_CANCELLED
|
||
and row.finished_at is not None
|
||
and row.duration_ms > 2 * 3600 * 1000
|
||
and row.message == "检查脚本收尾",
|
||
f"status={row.status} finished={row.finished_at} dur={row.duration_ms} msg={row.message!r}",
|
||
)
|
||
check("⑤ 已经跑完的记录一个字都不改", db.get(models.HubSyncRun, done).status == models.HubSyncRun.STATUS_SUCCESS)
|
||
|
||
# ③ 持着锁的时候不许写坏真在跑的那趟
|
||
held = add_run(db, models.HubSyncRun.STATUS_RUNNING, now - timedelta(hours=5))
|
||
hub_sync.cache.holder = "fake-holder" # type: ignore[attr-defined]
|
||
check("③ 有人持锁时 reap 不动手", hub_sync.reap_stale_runs(db) == 0)
|
||
check("③ 状态保持进行中", db.get(models.HubSyncRun, held).status == models.HubSyncRun.STATUS_RUNNING)
|
||
|
||
# ④ 「终止」点名要动就动,顺带把锁交还
|
||
result = hub_sync.terminate_sync(db, operator_name="检查脚本")
|
||
check("④ 终止把挂着的「进行中」都收尾", result["cancelled"] == 2, str(result))
|
||
check(
|
||
"④ 跨副本锁被放行",
|
||
result["lock_released"] is True and hub_sync.cache.holder == "", # type: ignore[attr-defined]
|
||
str(result),
|
||
)
|
||
check(
|
||
"④ 返回里是收尾后的样子",
|
||
bool(result["runs"]) and all(item["status"] == models.HubSyncRun.STATUS_CANCELLED for item in result["runs"]),
|
||
str(result["runs"]),
|
||
)
|
||
check("④ 终止之后不再算作运行中", hub_sync.running() is False)
|
||
check("④ 写上了是谁按的按钮", "检查脚本" in db.get(models.HubSyncRun, held).message)
|
||
|
||
# ④b 另一条线程交还本进程那把锁不许炸(同步在调度线程,请求在 worker 线程)
|
||
assert hub_sync._run_lock.acquire(blocking=False)
|
||
threading.Thread(target=lambda: hub_sync._release_sync_locks("local:fake")).start()
|
||
time.sleep(0.3)
|
||
locked_after = hub_sync._run_lock.acquire(blocking=False)
|
||
check("④ 别的线程交还本进程锁不炸", locked_after)
|
||
if locked_after:
|
||
hub_sync._run_lock.release()
|
||
|
||
# ⑥ 被「终止」的那趟如果还活着,真结果盖掉终止标记
|
||
live = add_run(db, models.HubSyncRun.STATUS_RUNNING, datetime.now())
|
||
hub_sync.terminate_sync(db, operator_name="检查脚本")
|
||
check("⑥ 终止先把它标成已终止", db.get(models.HubSyncRun, live).status == models.HubSyncRun.STATUS_CANCELLED)
|
||
stats = hub_sync.SyncStats()
|
||
stats.total_users = 7
|
||
hub_sync._finish(db, live, stats, models.HubSyncRun.STATUS_SUCCESS, "真跑完了")
|
||
row = db.get(models.HubSyncRun, live)
|
||
check(
|
||
"⑥ 真收尾把状态改回来",
|
||
row.status == models.HubSyncRun.STATUS_SUCCESS and row.total_users == 7,
|
||
f"status={row.status}",
|
||
)
|
||
|
||
# ⑦ 读一次状态就自愈
|
||
zombie = add_run(db, models.HubSyncRun.STATUS_RUNNING, datetime.now() - timedelta(hours=6))
|
||
payload = hub_sync.status_payload(db)
|
||
check("⑦ 读一次状态就收掉残留", db.get(models.HubSyncRun, zombie).status == models.HubSyncRun.STATUS_CANCELLED)
|
||
check("⑦ 状态接口说现在没在跑", payload["running"] is False)
|
||
|
||
# ⑧ 接口那条路:没登录 -> 401;管理员登录 -> 真把残留收尾并留操作日志
|
||
client = TestClient(app, raise_server_exceptions=False)
|
||
anon = client.post("/api/v1/hub/sync/cancel")
|
||
check("⑧ 没登录不给终止", anon.status_code == 401, f"{anon.status_code} {anon.text[:120]}")
|
||
|
||
admin = db.get(models.User, 1)
|
||
if admin is None:
|
||
print("[跳过] 库里没有 1 号用户,接口那一条校不了")
|
||
else:
|
||
token, _ = create_access_token(str(admin.id))
|
||
headers = {"Authorization": f"Bearer {token}"}
|
||
# 起步 5 分钟:还没到自动收尾那 30 分钟,所以接口会如实报「进行中」,
|
||
# 而「终止」按 grace=0 走,照样能把它收掉
|
||
target = add_run(db, models.HubSyncRun.STATUS_RUNNING, datetime.now() - timedelta(minutes=5))
|
||
check(
|
||
"⑧ 同步历史里那条如实显示「进行中」",
|
||
run_of(client, headers, target).get("status") == models.HubSyncRun.STATUS_RUNNING,
|
||
str(run_of(client, headers, target))[:160],
|
||
)
|
||
got = client.post("/api/v1/hub/sync/cancel", headers=headers)
|
||
data = (got.json() or {}).get("data") or {}
|
||
# 接口那趟是另一条事务:本会话还站在旧快照上(MySQL 默认可重复读),
|
||
# 读完之前先把它结束掉,否则看不见接口刚提交的东西
|
||
db.rollback()
|
||
check("⑧ 终止接口回 200", got.status_code == 200, got.text[:200])
|
||
check(
|
||
"⑧ 真把那条「进行中」收掉了",
|
||
int(data.get("cancelled") or 0) >= 1
|
||
and db.get(models.HubSyncRun, target).status == models.HubSyncRun.STATUS_CANCELLED
|
||
and run_of(client, headers, target).get("status") == models.HubSyncRun.STATUS_CANCELLED,
|
||
f"{str(data)[:160]} / {str(run_of(client, headers, target))[:120]}",
|
||
)
|
||
logged = db.execute(
|
||
select(models.OperationLog)
|
||
.where(
|
||
models.OperationLog.action == "hub_sync",
|
||
models.OperationLog.detail.like("手动终止统一身份同步%"),
|
||
)
|
||
.order_by(models.OperationLog.create_time.desc())
|
||
).scalars().first()
|
||
check(
|
||
"⑧ 留了一条操作日志",
|
||
logged is not None and "终止" in (logged.detail if logged else ""),
|
||
"没有匹配的操作日志" if logged is None else logged.detail[:160],
|
||
)
|
||
check(
|
||
"⑧ 终止之后状态接口不再说在跑",
|
||
((client.get("/api/v1/hub/status", headers=headers).json() or {}).get("data") or {}).get("running") is False,
|
||
)
|
||
|
||
# ⑨ 旧标记不许停新同步:终止是按"这一趟开始之后按过没有"算的
|
||
marked_at = hub_sync.request_cancel()
|
||
check("⑨ 按之前的那趟算被叫停", hub_sync.cancel_requested(marked_at - timedelta(minutes=5)))
|
||
check("⑨ 按之后重开的这趟不算", hub_sync.cancel_requested(marked_at + timedelta(seconds=1)) is False)
|
||
check("⑨ 没给开始时间时只看有没有按过", hub_sync.cancel_requested() is True)
|
||
|
||
# ⑩ 真跑一趟:假 Hub 在"拉目录"这几十秒里被人按了终止 => 停在检查点,什么都没写
|
||
real_client = hub_sync.hub_api.client
|
||
real_token = hub_sync.hub_api.sync_token
|
||
|
||
class StalledHub:
|
||
def directory(self, token, **kwargs): # noqa: ANN001, ANN003
|
||
hub_sync.request_cancel() # 就当作是页面上有人按了「终止」
|
||
return {
|
||
"permission_version": 1,
|
||
"total_users": 1,
|
||
"returned_users": 1,
|
||
"organizations": [],
|
||
"users": [{"user_id": 900000001, "account": MARK, "display_name": "终止校核", "status": 1}],
|
||
}
|
||
|
||
hub_sync.hub_api.client = lambda: StalledHub()
|
||
# 这一趟自己会建一条运行记录,operator 留空,清理时按"手动 + 这一分钟 + 已终止"认出来
|
||
hub_sync.hub_api.sync_token = lambda cli: ("token", "directory_key")
|
||
message = ""
|
||
try:
|
||
hub_sync.sync_users(db, trigger=models.HubSyncRun.TRIGGER_MANUAL, operator_name="检查脚本")
|
||
except hub_sync.HubSyncError as exc:
|
||
message = str(exc)
|
||
finally:
|
||
hub_sync.hub_api.client = real_client
|
||
hub_sync.hub_api.sync_token = real_token
|
||
check("⑩ 跑起来的同步被叫停在检查点", "终止" in message, message)
|
||
stopped = hub_sync.latest_run(db)
|
||
check(
|
||
"⑩ 记录写的是「已终止」,不是「失败」",
|
||
stopped is not None and stopped.status == models.HubSyncRun.STATUS_CANCELLED,
|
||
f"{getattr(stopped, 'status', None)} / {message}",
|
||
)
|
||
check("⑩ 叫停之后不再算作运行中", hub_sync.running() is False)
|
||
check("⑩ 假 Hub 没往库里写人", db.execute(
|
||
select(models.User).where(models.User.account == MARK)
|
||
).scalars().first() is None)
|
||
finally:
|
||
cleanup = SessionLocal()
|
||
try:
|
||
ids = list(
|
||
cleanup.execute(select(models.HubSyncRun.id).where(models.HubSyncRun.operator == MARK)).scalars().all()
|
||
)
|
||
if ids:
|
||
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(ids)))
|
||
cleanup.commit()
|
||
mine = list(
|
||
cleanup.execute(
|
||
select(models.HubSyncRun.id).where(
|
||
models.HubSyncRun.trigger == models.HubSyncRun.TRIGGER_MANUAL,
|
||
models.HubSyncRun.operator_name == "检查脚本",
|
||
)
|
||
).scalars().all()
|
||
)
|
||
if mine:
|
||
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(mine)))
|
||
cleanup.commit()
|
||
ids += [row_id for row_id in mine if row_id not in ids]
|
||
logged = list(
|
||
cleanup.execute(
|
||
select(models.OperationLog.id).where(
|
||
models.OperationLog.action == "hub_sync",
|
||
models.OperationLog.detail.like("手动终止统一身份同步%"),
|
||
)
|
||
).scalars().all()
|
||
)
|
||
if logged:
|
||
cleanup.execute(delete(models.OperationLog).where(models.OperationLog.id.in_(logged)))
|
||
cleanup.commit()
|
||
print(f"[清理] 删掉 {len(ids)} 条临时记录、{len(logged)} 条操作日志")
|
||
finally:
|
||
cleanup.close()
|
||
hub_sync.cache = real_cache # type: ignore[assignment]
|
||
db.close()
|
||
|
||
print()
|
||
if FAILS:
|
||
print(f"未通过 {len(FAILS)} 项:")
|
||
for name in FAILS:
|
||
print(f" - {name}")
|
||
return 1
|
||
print(f"全部通过({PASSED} 项):同步卡住既能自己收尾,也能由人按下「终止」带走。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|