unis-oa/backend/scripts/check_notify_wake.py

257 lines
11 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
"""渠道提醒 / 订阅事件什么时候真的发出去:入队提交当场叫醒,回滚不空叫。
cd backend && PYTHONPATH=. python3 scripts/check_notify_wake.py
开发库上 ``oa_notify_delivery`` 的量法:一条企业微信推送从 ``create_time`` 到
``sent_at`` 等了 **28 / 35 / 55 / 60 秒**,而 ``ms`` 那一格显示真发那一次只用 1 秒。
慢的不是发送——是消息渠道与事件订阅入队时**没人叫醒**调度线程,只能干等下一个轮询
周期(写回发件箱早就有叫醒,那两路没接上)。这几条把叫醒的时机钉住:
1. **随事务提交当场叫**:``channels.enqueue_for`` 排上投递记录,提交那一刻叫醒;
2. **没提交不叫**:事务还没提交,那条记录对别的连接还不可见,叫了是跑一趟空;
3. **回滚不叫**:单没过,投递记录跟着一起回滚;
4. **没排上行不叫**:事件没渠道订阅 → 一条记录都没有 → 不空叫;
5. **订阅事件同一条通路**:``events.emit`` 入队也是提交即叫;
6. **人工重发也叫**:点「重发」的人就是想看结果。
渠道与订阅都指向 ``*.invalid``(DNS 必然失败),跑完把渠道、订阅、投递记录、事件删干净。
"""
from __future__ import annotations
import pathlib
import sys
import time
from datetime import datetime, timedelta
from types import SimpleNamespace
HERE = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from sqlalchemy import delete, or_, select # noqa: E402
from app import models # noqa: E402
from app.db import SessionLocal # noqa: E402
from app.service import channels, events, wake # noqa: E402
STAMP = int(time.time())
CH = f"wk_ch_{STAMP}"
HOOK = f"wk_hook_{STAMP}"
JOB = f"wkwakejob{STAMP}"
#: 测试自己造的事件名,**不用 task / work.submitted**:开发库里躺着真的渠道与订阅,
#: 拿真事件名入队会给它们也排上一条,叫醒机制一生效就真的往企业微信发出去了。
EVENT = f"wk_event_{STAMP}"
EVENT_NONE = f"wk_none_{STAMP}"
EVENT_OPEN = f"wk_open_{STAMP}"
EVENT_OPEN_NONE = f"wk_open_none_{STAMP}"
TITLE = f"[叫醒校验 {STAMP}]"
FAILS: list[str] = []
PASSED = 0
WAKES: list[int] = []
CREATED: dict = {}
#: 把刚排上的记录推到将来,别让**同时在跑的那个后端进程**的调度线程抢发(它们会
#: 真的去连 ``*.invalid``,多一条没意义的失败留痕;测的是"什么时候醒",不是发得成)
PARK_SECONDS = 600
def check(label: str, condition, detail: str = "") -> None:
global PASSED
if condition:
PASSED += 1
print(f"[通过] {label}")
else:
FAILS.append(label)
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
def park(db, rows) -> None:
moment = (datetime.now() + timedelta(seconds=PARK_SECONDS)).replace(microsecond=0)
for row in rows:
row.next_attempt_at = moment
db.flush()
db.commit()
def deliveries(db) -> list[models.NotifyDelivery]:
return list(db.execute(
select(models.NotifyDelivery).where(models.NotifyDelivery.channel == CH)
).scalars().all())
def build(db) -> str:
"""一个真账号 + 一条 webhook 渠道 + 一条订阅,地址都指向打不通的域名。"""
person = db.execute(
select(models.User)
.where(models.User.status.in_(sorted(models.UserStatus.LOGIN_ALLOWED)))
.order_by(models.User.id)
.limit(1)
).scalars().first()
if person is None:
raise SystemExit("[跳过] 库里没有一个可登录的账号,没法验收件人")
db.add(models.NotifyChannel(
code=CH, name=f"{TITLE} 渠道", kind=models.NotifyChannel.KIND_WEBHOOK,
enabled=True, events=[EVENT], config={"url": "https://robot.invalid/hook"},
timeout_seconds=2, retries=1,
))
db.add(models.OpenWebhook(
code=HOOK, name=f"{TITLE} 订阅", enabled=True,
url="https://sub.invalid/hook", events=[EVENT_OPEN], send_data=False,
timeout_seconds=2, retries=1,
))
db.commit()
# presence 那一层可能还存着"这张表没有启用的行",本 Session 写过就该照实问库
CREATED.update(channel=CH, hook=HOOK)
return str(person.account)
def fake_work() -> SimpleNamespace:
return SimpleNamespace(job=JOB, id=JOB, serial=f"WKWAKE{STAMP}", title="叫醒校验单",
status="running", process="", process_name="", creator="1",
creator_person="1", activity="n1", data={})
def _run(db) -> None:
account = build(db)
wake.hook(lambda: WAKES.append(len(WAKES)))
check("渠道建好就能被投递侧看到(presence 没把'没有渠道'这个旧结论留住)",
any(row.code == CH for row in channels.channels_for(db, EVENT)),
[row.code for row in channels.channels_for(db, EVENT)])
# ---------------------------------------------- 1/2. 提交才算数,提交那一刻才叫
rows = channels.enqueue_for(db, [account], event=EVENT, title=f"{TITLE} 待办",
body="验证入队叫醒", job=JOB)
db.flush()
check("排上了投递记录(收件人认得出、渠道订阅了这个事件)", rows >= 1, rows)
check("事务没提交时不叫醒(那条记录对别的连接还不可见,叫了是跑空)", not WAKES, WAKES)
db.commit()
check("投递记录随事务提交 → 当场叫醒投递线程", len(WAKES) == 1, WAKES)
park(db, deliveries(db))
# ---------------------------------------------- 3. 回滚不叫
WAKES.clear()
before = len(deliveries(db))
channels.enqueue_for(db, [account], event=EVENT, title=f"{TITLE} 回滚不算数", job=JOB)
db.flush()
db.rollback()
check("事务回滚了不叫醒(单没过,投递记录也跟着没了)", not WAKES, WAKES)
check("回滚之后库里没多出投递记录", len(deliveries(db)) == before,
(before, len(deliveries(db))))
# ---------------------------------------------- 4. 没排上行不空叫
WAKES.clear()
rows = channels.enqueue_for(db, [account], event=EVENT_NONE, title=f"{TITLE} 没人订阅", job=JOB)
db.commit()
check("这个事件没渠道订阅:一条都不排,也不空叫一次", rows == 0 and not WAKES, (rows, WAKES))
# ---------------------------------------------- 5. 订阅事件走同一条通路
WAKES.clear()
count = events.emit(db, fake_work(), EVENT_OPEN, activity_name="提交")
db.flush()
check("订阅入队时先不叫(等提交)", count >= 1 and not WAKES, (count, WAKES))
db.commit()
check("事件订阅随事务提交 → 当场叫醒", len(WAKES) == 1, WAKES)
WAKES.clear()
count = events.emit(db, fake_work(), EVENT_OPEN_NONE, activity_name="作废")
db.commit()
check("没人订阅这个事件:不排、不叫", count == 0 and not WAKES, (count, WAKES))
# ---------------------------------------------- 6. 人工重发也叫醒
WAKES.clear()
rows = deliveries(db)
check("前面这几趟留下了可重发的投递记录", bool(rows), len(rows))
if rows:
resent = channels.resend(db, rows[0].id)
check("人工重发:回到 pending 并当场叫醒(点重发的人就是想看结果)",
resent is not None and resent.status == models.NotifyDelivery.STATUS_PENDING
and len(WAKES) == 1,
(getattr(resent, "status", None), WAKES))
if resent is not None:
park(db, [resent])
# ---------------------------------------------- 7. 叫醒只改"何时醒",不改跑什么
from app import main as main_module
stir, stop = main_module._scheduler_stir, main_module._scheduler_stop
saved_stop = stop.is_set()
stop.clear()
stir.clear()
started = time.monotonic()
main_module._wake_scheduler()
check("被叫醒时 _scheduler_sleep 立刻返回,且不退出线程",
main_module._scheduler_sleep(30) is False and time.monotonic() - started < 5,
f"等了 {time.monotonic() - started:.2f}s")
stop.set()
stir.clear()
started = time.monotonic()
main_module._wake_scheduler()
check("退出信号优先:睡里被叫醒也要认出该退出",
main_module._scheduler_sleep(30) is True and time.monotonic() - started < 5,
f"等了 {time.monotonic() - started:.2f}s")
if saved_stop:
stop.set()
else:
stop.clear()
stir.clear()
# ---------------------------------------------- 8. 接线:调度线程真的在听这一层
# 上面那几条验的是"入队 → 该叫",这一段验"叫了 → 线程真的醒":
# ``start_scheduler`` 里那句登记把 ``_wake_scheduler`` 接到本层。这里把 Thread 换成
# 不起动的假类,只验接线,不在校验进程里真跑一轮调度(那会去抢租约、去发 parked 的行)。
real_thread = main_module.threading.Thread
class QuietThread(real_thread):
def __init__(self, *args, **kwargs): # 记下参数,但不起线程
self.args, self.kwargs = args, kwargs
def start(self):
return None
main_module.threading.Thread = QuietThread
try:
main_module.start_scheduler()
finally:
main_module.threading.Thread = real_thread
check("start_scheduler 把调度线程登记成叫醒对象(渠道与事件共享写回那条通路)",
main_module._wake_scheduler in wake._hooks, len(wake._hooks))
main_module._scheduler_stir.clear()
wake.now()
check("叫醒通路 → 调度线程那条睡真的被中断", main_module._scheduler_stir.is_set())
main_module._scheduler_stir.clear()
def main() -> int:
db = SessionLocal()
try:
_run(db)
finally:
cleanup(db)
print()
if FAILS:
print(f"{len(FAILS)} 条没对上:")
for item in FAILS:
print(f" - {item}")
else:
print(f"全部 {PASSED} 条对上。")
return 1 if FAILS else 0
def cleanup(db) -> None:
db.execute(delete(models.NotifyDelivery).where(or_(
models.NotifyDelivery.channel == CH,
models.NotifyDelivery.title.like(f"{TITLE}%"))))
db.execute(delete(models.OpenEvent).where(or_(models.OpenEvent.webhook == HOOK,
models.OpenEvent.job == JOB)))
db.execute(delete(models.OpenWebhook).where(models.OpenWebhook.code == HOOK))
db.execute(delete(models.NotifyChannel).where(models.NotifyChannel.code == CH))
db.commit()
print("[清理] 渠道、订阅、投递记录与事件已删干净")
if __name__ == "__main__":
raise SystemExit(main())