unis-oa/backend/scripts/check_hub_sync_cancel.py

366 lines
17 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
"""守「终止同步」这条路:卡住的「进行中」必须出得去,真在跑的不许被写坏。
踩过的坑:一趟同步刚开始时后端被 kill(重启、发布、容器驱逐),
``oa_hub_sync_run`` 里那条 ``status='running'`` 就再也没人收尾 —— 同步历史一直显示
「进行中」,页头「立即同步」的 ``busy`` 也跟着永远为真、按钮按不动,只能连库手改。
这一层给两条出路:启动 / 读状态时自动收尾残留
(:func:`app.service.hub_sync.reap_stale_runs`),以及界面上那个人按的「终止」
(:func:`app.service.hub_sync.terminate_sync` + ``POST /api/v1/hub/sync/cancel``)。
这里钉七条::
cd backend && PYTHONPATH=. python3 scripts/check_hub_sync_cancel.py
1. **没到点的不算残留**:刚起步一分钟的「进行中」不动它,否则一趟正常的慢同步会被
谁读一次状态就写坏;
2. **到点又没人持锁才收尾**:超过宽限期且没有同步锁的,标成「已终止」、补结束时间与耗时;
3. **持着锁就一个字都不改**:真在跑的那趟不许被状态接口写坏;
4. **「终止」点名要动就动**:``grace_seconds=0`` 连刚起步的记录也收尾,并把锁交还;
跑在另一条线程上的那一趟随后自己 release 也不许炸;
5. **已经结束的永远不碰**:success / failed / skipped 改一条都是伪造历史;
6. **收尾不吞真结果**:被「终止」的那趟如果其实还活着,自己收尾时要把状态改回真实结果;
7. **读一次状态就自愈**:状态接口顺带收残留,页面不必等后端重启;
8. **接口那条路走得通**:``POST /api/v1/hub/sync/cancel`` 挂着 ``org:person``,没登录回
401,登录后真把库里那条「进行中」收尾,并留一条操作日志(``action=hub_sync``);
9. **旧标记不许停新同步**:终止信号按"这一趟开始之后有没有人按过"算,刚终止完立刻
重开的那一趟不能被上一趟的标记停掉;
10. **真跑起来的那一趟会被叫停**:拉目录期间按下终止,整趟停在检查点上,记录写
「已终止」而不是「失败」,一个部门、一个人都没往库里写。
不碰真 Redis(换成假锁,免得 force_release 删掉线上正拿着的那把),库里只写自己造的
临时行,跑完按标记删干净。
"""
from __future__ import annotations
import pathlib
import sys
import threading
import time
from datetime import datetime, timedelta
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
from sqlalchemy import delete, select # noqa: E402
from starlette.testclient import TestClient # noqa: E402
from app import models # noqa: E402
from app.db import SessionLocal # noqa: E402
from app.main import app # noqa: E402
from app.security import create_access_token # noqa: E402
from app.service import hub_sync # noqa: E402
MARK = "zz-check-hub-cancel"
FAILS: list[str] = []
PASSED = 0
class FakeCache:
"""假锁 + 假取值层:只记"有没有人拿着"和几个键。真 Redis 上 force_release 会删掉
别人正拿着的锁,终止标记也会写到别的副本脸上,校验收脚本不该干这事。"""
def __init__(self) -> None:
self.holder = ""
self.store: dict[str, object] = {}
def key(self, *parts) -> str: # noqa: ANN002
return ":".join(str(part) for part in parts)
def set_json(self, name: str, value: object, ttl_seconds: float) -> bool:
self.store[name] = value
return True
def get_json(self, name: str): # noqa: ANN201
return self.store.get(name)
def delete(self, *names) -> int: # noqa: ANN002
gone = 0
for name in names:
if name in self.store:
self.store.pop(name)
gone += 1
return gone
def lock_holder(self, name: str) -> str:
return self.holder
def locked(self, name: str) -> bool:
return bool(self.holder)
def try_lock(self, name: str, *, ttl_seconds: float) -> str:
if self.holder:
return ""
self.holder = f"fake-{threading.get_ident()}"
return self.holder
def release_lock(self, name: str, token: str) -> None:
if token and self.holder == token:
self.holder = ""
def force_release(self, name: str) -> bool:
had = bool(self.holder)
self.holder = ""
return had
def check(label: str, passed: bool, detail: str = "") -> None:
global PASSED
if passed:
PASSED += 1
print(f"[通过] {label}")
else:
FAILS.append(label)
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
def add_run(db, status: str, started_at: datetime, *, minutes_ago: float = 0): # noqa: ANN001
row = models.HubSyncRun(
trigger=models.HubSyncRun.TRIGGER_CHANGES,
status=status,
started_at=started_at,
operator=MARK,
)
db.add(row)
db.commit()
return row.id
def run_of(client, headers, run_id: str) -> dict: # noqa: ANN001
"""从 ``GET /hub/sync/runs`` 里取自己造的那一条(``last_run`` 未必是它)。"""
payload = client.get("/api/v1/hub/sync/runs", params={"limit": 50}, headers=headers).json()
for row in (payload or {}).get("data") or []:
if row.get("id") == run_id:
return row
return {}
def main() -> int:
real_cache = hub_sync.cache
hub_sync.cache = FakeCache() # type: ignore[assignment]
db = SessionLocal()
try:
now = datetime.now()
# ① 没到点的不算残留
fresh = add_run(db, models.HubSyncRun.STATUS_RUNNING, now - timedelta(minutes=1))
# 这库是共享的,别的环境留下的残留会被这一次顺带收掉,所以只验自己那条没动
hub_sync.reap_stale_runs(db)
check(
"① 刚起步的「进行中」不被自动收尾",
db.get(models.HubSyncRun, fresh).status == models.HubSyncRun.STATUS_RUNNING,
)
# ② 到点且没人持锁 => 收尾;⑤ 已经结束的不动
old = add_run(db, models.HubSyncRun.STATUS_RUNNING, now - timedelta(hours=3))
done = add_run(db, models.HubSyncRun.STATUS_SUCCESS, now - timedelta(hours=4))
row = db.get(models.HubSyncRun, done)
row.finished_at = now - timedelta(hours=4, minutes=2)
db.commit()
check("② 到点的残留被收尾", hub_sync.reap_stale_runs(db, message="检查脚本收尾") == 1)
row = db.get(models.HubSyncRun, old)
check(
"② 收尾补齐状态/结束时间/耗时",
row.status == models.HubSyncRun.STATUS_CANCELLED
and row.finished_at is not None
and row.duration_ms > 2 * 3600 * 1000
and row.message == "检查脚本收尾",
f"status={row.status} finished={row.finished_at} dur={row.duration_ms} msg={row.message!r}",
)
check("⑤ 已经跑完的记录一个字都不改", db.get(models.HubSyncRun, done).status == models.HubSyncRun.STATUS_SUCCESS)
# ③ 持着锁的时候不许写坏真在跑的那趟
held = add_run(db, models.HubSyncRun.STATUS_RUNNING, now - timedelta(hours=5))
hub_sync.cache.holder = "fake-holder" # type: ignore[attr-defined]
check("③ 有人持锁时 reap 不动手", hub_sync.reap_stale_runs(db) == 0)
check("③ 状态保持进行中", db.get(models.HubSyncRun, held).status == models.HubSyncRun.STATUS_RUNNING)
# ④ 「终止」点名要动就动,顺带把锁交还
result = hub_sync.terminate_sync(db, operator_name="检查脚本")
check("④ 终止把挂着的「进行中」都收尾", result["cancelled"] == 2, str(result))
check(
"④ 跨副本锁被放行",
result["lock_released"] is True and hub_sync.cache.holder == "", # type: ignore[attr-defined]
str(result),
)
check(
"④ 返回里是收尾后的样子",
bool(result["runs"]) and all(item["status"] == models.HubSyncRun.STATUS_CANCELLED for item in result["runs"]),
str(result["runs"]),
)
check("④ 终止之后不再算作运行中", hub_sync.running() is False)
check("④ 写上了是谁按的按钮", "检查脚本" in db.get(models.HubSyncRun, held).message)
# ④b 另一条线程交还本进程那把锁不许炸(同步在调度线程,请求在 worker 线程)
assert hub_sync._run_lock.acquire(blocking=False)
threading.Thread(target=lambda: hub_sync._release_sync_locks("local:fake")).start()
time.sleep(0.3)
locked_after = hub_sync._run_lock.acquire(blocking=False)
check("④ 别的线程交还本进程锁不炸", locked_after)
if locked_after:
hub_sync._run_lock.release()
# ⑥ 被「终止」的那趟如果还活着,真结果盖掉终止标记
live = add_run(db, models.HubSyncRun.STATUS_RUNNING, datetime.now())
hub_sync.terminate_sync(db, operator_name="检查脚本")
check("⑥ 终止先把它标成已终止", db.get(models.HubSyncRun, live).status == models.HubSyncRun.STATUS_CANCELLED)
stats = hub_sync.SyncStats()
stats.total_users = 7
hub_sync._finish(db, live, stats, models.HubSyncRun.STATUS_SUCCESS, "真跑完了")
row = db.get(models.HubSyncRun, live)
check(
"⑥ 真收尾把状态改回来",
row.status == models.HubSyncRun.STATUS_SUCCESS and row.total_users == 7,
f"status={row.status}",
)
# ⑦ 读一次状态就自愈
zombie = add_run(db, models.HubSyncRun.STATUS_RUNNING, datetime.now() - timedelta(hours=6))
payload = hub_sync.status_payload(db)
check("⑦ 读一次状态就收掉残留", db.get(models.HubSyncRun, zombie).status == models.HubSyncRun.STATUS_CANCELLED)
check("⑦ 状态接口说现在没在跑", payload["running"] is False)
# ⑧ 接口那条路:没登录 -> 401;管理员登录 -> 真把残留收尾并留操作日志
client = TestClient(app, raise_server_exceptions=False)
anon = client.post("/api/v1/hub/sync/cancel")
check("⑧ 没登录不给终止", anon.status_code == 401, f"{anon.status_code} {anon.text[:120]}")
admin = db.get(models.User, 1)
if admin is None:
print("[跳过] 库里没有 1 号用户,接口那一条校不了")
else:
token, _ = create_access_token(str(admin.id))
headers = {"Authorization": f"Bearer {token}"}
# 起步 5 分钟:还没到自动收尾那 30 分钟,所以接口会如实报「进行中」,
# 而「终止」按 grace=0 走,照样能把它收掉
target = add_run(db, models.HubSyncRun.STATUS_RUNNING, datetime.now() - timedelta(minutes=5))
check(
"⑧ 同步历史里那条如实显示「进行中」",
run_of(client, headers, target).get("status") == models.HubSyncRun.STATUS_RUNNING,
str(run_of(client, headers, target))[:160],
)
got = client.post("/api/v1/hub/sync/cancel", headers=headers)
data = (got.json() or {}).get("data") or {}
# 接口那趟是另一条事务:本会话还站在旧快照上(MySQL 默认可重复读),
# 读完之前先把它结束掉,否则看不见接口刚提交的东西
db.rollback()
check("⑧ 终止接口回 200", got.status_code == 200, got.text[:200])
check(
"⑧ 真把那条「进行中」收掉了",
int(data.get("cancelled") or 0) >= 1
and db.get(models.HubSyncRun, target).status == models.HubSyncRun.STATUS_CANCELLED
and run_of(client, headers, target).get("status") == models.HubSyncRun.STATUS_CANCELLED,
f"{str(data)[:160]} / {str(run_of(client, headers, target))[:120]}",
)
logged = db.execute(
select(models.OperationLog)
.where(
models.OperationLog.action == "hub_sync",
models.OperationLog.detail.like("手动终止统一身份同步%"),
)
.order_by(models.OperationLog.create_time.desc())
).scalars().first()
check(
"⑧ 留了一条操作日志",
logged is not None and "终止" in (logged.detail if logged else ""),
"没有匹配的操作日志" if logged is None else logged.detail[:160],
)
check(
"⑧ 终止之后状态接口不再说在跑",
((client.get("/api/v1/hub/status", headers=headers).json() or {}).get("data") or {}).get("running") is False,
)
# ⑨ 旧标记不许停新同步:终止是按"这一趟开始之后按过没有"算的
marked_at = hub_sync.request_cancel()
check("⑨ 按之前的那趟算被叫停", hub_sync.cancel_requested(marked_at - timedelta(minutes=5)))
check("⑨ 按之后重开的这趟不算", hub_sync.cancel_requested(marked_at + timedelta(seconds=1)) is False)
check("⑨ 没给开始时间时只看有没有按过", hub_sync.cancel_requested() is True)
# ⑩ 真跑一趟:假 Hub 在"拉目录"这几十秒里被人按了终止 => 停在检查点,什么都没写
real_client = hub_sync.hub_api.client
real_token = hub_sync.hub_api.sync_token
class StalledHub:
def directory(self, token, **kwargs): # noqa: ANN001, ANN003
hub_sync.request_cancel() # 就当作是页面上有人按了「终止」
return {
"permission_version": 1,
"total_users": 1,
"returned_users": 1,
"organizations": [],
"users": [{"user_id": 900000001, "account": MARK, "display_name": "终止校核", "status": 1}],
}
hub_sync.hub_api.client = lambda: StalledHub()
# 这一趟自己会建一条运行记录,operator 留空,清理时按"手动 + 这一分钟 + 已终止"认出来
hub_sync.hub_api.sync_token = lambda cli: ("token", "directory_key")
message = ""
try:
hub_sync.sync_users(db, trigger=models.HubSyncRun.TRIGGER_MANUAL, operator_name="检查脚本")
except hub_sync.HubSyncError as exc:
message = str(exc)
finally:
hub_sync.hub_api.client = real_client
hub_sync.hub_api.sync_token = real_token
check("⑩ 跑起来的同步被叫停在检查点", "终止" in message, message)
stopped = hub_sync.latest_run(db)
check(
"⑩ 记录写的是「已终止」,不是「失败」",
stopped is not None and stopped.status == models.HubSyncRun.STATUS_CANCELLED,
f"{getattr(stopped, 'status', None)} / {message}",
)
check("⑩ 叫停之后不再算作运行中", hub_sync.running() is False)
check("⑩ 假 Hub 没往库里写人", db.execute(
select(models.User).where(models.User.account == MARK)
).scalars().first() is None)
finally:
cleanup = SessionLocal()
try:
ids = list(
cleanup.execute(select(models.HubSyncRun.id).where(models.HubSyncRun.operator == MARK)).scalars().all()
)
if ids:
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(ids)))
cleanup.commit()
mine = list(
cleanup.execute(
select(models.HubSyncRun.id).where(
models.HubSyncRun.trigger == models.HubSyncRun.TRIGGER_MANUAL,
models.HubSyncRun.operator_name == "检查脚本",
)
).scalars().all()
)
if mine:
cleanup.execute(delete(models.HubSyncRun).where(models.HubSyncRun.id.in_(mine)))
cleanup.commit()
ids += [row_id for row_id in mine if row_id not in ids]
logged = list(
cleanup.execute(
select(models.OperationLog.id).where(
models.OperationLog.action == "hub_sync",
models.OperationLog.detail.like("手动终止统一身份同步%"),
)
).scalars().all()
)
if logged:
cleanup.execute(delete(models.OperationLog).where(models.OperationLog.id.in_(logged)))
cleanup.commit()
print(f"[清理] 删掉 {len(ids)} 条临时记录、{len(logged)} 条操作日志")
finally:
cleanup.close()
hub_sync.cache = real_cache # type: ignore[assignment]
db.close()
print()
if FAILS:
print(f"未通过 {len(FAILS)} 项:")
for name in FAILS:
print(f" - {name}")
return 1
print(f"全部通过({PASSED} 项):同步卡住既能自己收尾,也能由人按下「终止」带走。")
return 0
if __name__ == "__main__":
raise SystemExit(main())