257 lines
11 KiB
Python
257 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""渠道提醒 / 订阅事件什么时候真的发出去:入队提交当场叫醒,回滚不空叫。
|
||
|
||
cd backend && PYTHONPATH=. python3 scripts/check_notify_wake.py
|
||
|
||
开发库上 ``oa_notify_delivery`` 的量法:一条企业微信推送从 ``create_time`` 到
|
||
``sent_at`` 等了 **28 / 35 / 55 / 60 秒**,而 ``ms`` 那一格显示真发那一次只用 1 秒。
|
||
慢的不是发送——是消息渠道与事件订阅入队时**没人叫醒**调度线程,只能干等下一个轮询
|
||
周期(写回发件箱早就有叫醒,那两路没接上)。这几条把叫醒的时机钉住:
|
||
|
||
1. **随事务提交当场叫**:``channels.enqueue_for`` 排上投递记录,提交那一刻叫醒;
|
||
2. **没提交不叫**:事务还没提交,那条记录对别的连接还不可见,叫了是跑一趟空;
|
||
3. **回滚不叫**:单没过,投递记录跟着一起回滚;
|
||
4. **没排上行不叫**:事件没渠道订阅 → 一条记录都没有 → 不空叫;
|
||
5. **订阅事件同一条通路**:``events.emit`` 入队也是提交即叫;
|
||
6. **人工重发也叫**:点「重发」的人就是想看结果。
|
||
|
||
渠道与订阅都指向 ``*.invalid``(DNS 必然失败),跑完把渠道、订阅、投递记录、事件删干净。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pathlib
|
||
import sys
|
||
import time
|
||
from datetime import datetime, timedelta
|
||
from types import SimpleNamespace
|
||
|
||
HERE = pathlib.Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(HERE))
|
||
sys.path.insert(0, str(HERE.parent))
|
||
|
||
from sqlalchemy import delete, or_, select # noqa: E402
|
||
|
||
from app import models # noqa: E402
|
||
from app.db import SessionLocal # noqa: E402
|
||
from app.service import channels, events, wake # noqa: E402
|
||
|
||
STAMP = int(time.time())
|
||
CH = f"wk_ch_{STAMP}"
|
||
HOOK = f"wk_hook_{STAMP}"
|
||
JOB = f"wkwakejob{STAMP}"
|
||
#: 测试自己造的事件名,**不用 task / work.submitted**:开发库里躺着真的渠道与订阅,
|
||
#: 拿真事件名入队会给它们也排上一条,叫醒机制一生效就真的往企业微信发出去了。
|
||
EVENT = f"wk_event_{STAMP}"
|
||
EVENT_NONE = f"wk_none_{STAMP}"
|
||
EVENT_OPEN = f"wk_open_{STAMP}"
|
||
EVENT_OPEN_NONE = f"wk_open_none_{STAMP}"
|
||
TITLE = f"[叫醒校验 {STAMP}]"
|
||
|
||
FAILS: list[str] = []
|
||
PASSED = 0
|
||
WAKES: list[int] = []
|
||
CREATED: dict = {}
|
||
|
||
#: 把刚排上的记录推到将来,别让**同时在跑的那个后端进程**的调度线程抢发(它们会
|
||
#: 真的去连 ``*.invalid``,多一条没意义的失败留痕;测的是"什么时候醒",不是发得成)
|
||
PARK_SECONDS = 600
|
||
|
||
|
||
def check(label: str, condition, detail: str = "") -> None:
|
||
global PASSED
|
||
if condition:
|
||
PASSED += 1
|
||
print(f"[通过] {label}")
|
||
else:
|
||
FAILS.append(label)
|
||
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
|
||
|
||
|
||
def park(db, rows) -> None:
|
||
moment = (datetime.now() + timedelta(seconds=PARK_SECONDS)).replace(microsecond=0)
|
||
for row in rows:
|
||
row.next_attempt_at = moment
|
||
db.flush()
|
||
db.commit()
|
||
|
||
|
||
def deliveries(db) -> list[models.NotifyDelivery]:
|
||
return list(db.execute(
|
||
select(models.NotifyDelivery).where(models.NotifyDelivery.channel == CH)
|
||
).scalars().all())
|
||
|
||
|
||
def build(db) -> str:
|
||
"""一个真账号 + 一条 webhook 渠道 + 一条订阅,地址都指向打不通的域名。"""
|
||
person = db.execute(
|
||
select(models.User)
|
||
.where(models.User.status.in_(sorted(models.UserStatus.LOGIN_ALLOWED)))
|
||
.order_by(models.User.id)
|
||
.limit(1)
|
||
).scalars().first()
|
||
if person is None:
|
||
raise SystemExit("[跳过] 库里没有一个可登录的账号,没法验收件人")
|
||
db.add(models.NotifyChannel(
|
||
code=CH, name=f"{TITLE} 渠道", kind=models.NotifyChannel.KIND_WEBHOOK,
|
||
enabled=True, events=[EVENT], config={"url": "https://robot.invalid/hook"},
|
||
timeout_seconds=2, retries=1,
|
||
))
|
||
db.add(models.OpenWebhook(
|
||
code=HOOK, name=f"{TITLE} 订阅", enabled=True,
|
||
url="https://sub.invalid/hook", events=[EVENT_OPEN], send_data=False,
|
||
timeout_seconds=2, retries=1,
|
||
))
|
||
db.commit()
|
||
# presence 那一层可能还存着"这张表没有启用的行",本 Session 写过就该照实问库
|
||
CREATED.update(channel=CH, hook=HOOK)
|
||
return str(person.account)
|
||
|
||
|
||
def fake_work() -> SimpleNamespace:
|
||
return SimpleNamespace(job=JOB, id=JOB, serial=f"WKWAKE{STAMP}", title="叫醒校验单",
|
||
status="running", process="", process_name="", creator="1",
|
||
creator_person="1", activity="n1", data={})
|
||
|
||
|
||
def _run(db) -> None:
|
||
account = build(db)
|
||
wake.hook(lambda: WAKES.append(len(WAKES)))
|
||
check("渠道建好就能被投递侧看到(presence 没把'没有渠道'这个旧结论留住)",
|
||
any(row.code == CH for row in channels.channels_for(db, EVENT)),
|
||
[row.code for row in channels.channels_for(db, EVENT)])
|
||
|
||
# ---------------------------------------------- 1/2. 提交才算数,提交那一刻才叫
|
||
rows = channels.enqueue_for(db, [account], event=EVENT, title=f"{TITLE} 待办",
|
||
body="验证入队叫醒", job=JOB)
|
||
db.flush()
|
||
check("排上了投递记录(收件人认得出、渠道订阅了这个事件)", rows >= 1, rows)
|
||
check("事务没提交时不叫醒(那条记录对别的连接还不可见,叫了是跑空)", not WAKES, WAKES)
|
||
db.commit()
|
||
check("投递记录随事务提交 → 当场叫醒投递线程", len(WAKES) == 1, WAKES)
|
||
park(db, deliveries(db))
|
||
|
||
# ---------------------------------------------- 3. 回滚不叫
|
||
WAKES.clear()
|
||
before = len(deliveries(db))
|
||
channels.enqueue_for(db, [account], event=EVENT, title=f"{TITLE} 回滚不算数", job=JOB)
|
||
db.flush()
|
||
db.rollback()
|
||
check("事务回滚了不叫醒(单没过,投递记录也跟着没了)", not WAKES, WAKES)
|
||
check("回滚之后库里没多出投递记录", len(deliveries(db)) == before,
|
||
(before, len(deliveries(db))))
|
||
|
||
# ---------------------------------------------- 4. 没排上行不空叫
|
||
WAKES.clear()
|
||
rows = channels.enqueue_for(db, [account], event=EVENT_NONE, title=f"{TITLE} 没人订阅", job=JOB)
|
||
db.commit()
|
||
check("这个事件没渠道订阅:一条都不排,也不空叫一次", rows == 0 and not WAKES, (rows, WAKES))
|
||
|
||
# ---------------------------------------------- 5. 订阅事件走同一条通路
|
||
WAKES.clear()
|
||
count = events.emit(db, fake_work(), EVENT_OPEN, activity_name="提交")
|
||
db.flush()
|
||
check("订阅入队时先不叫(等提交)", count >= 1 and not WAKES, (count, WAKES))
|
||
db.commit()
|
||
check("事件订阅随事务提交 → 当场叫醒", len(WAKES) == 1, WAKES)
|
||
|
||
WAKES.clear()
|
||
count = events.emit(db, fake_work(), EVENT_OPEN_NONE, activity_name="作废")
|
||
db.commit()
|
||
check("没人订阅这个事件:不排、不叫", count == 0 and not WAKES, (count, WAKES))
|
||
|
||
# ---------------------------------------------- 6. 人工重发也叫醒
|
||
WAKES.clear()
|
||
rows = deliveries(db)
|
||
check("前面这几趟留下了可重发的投递记录", bool(rows), len(rows))
|
||
if rows:
|
||
resent = channels.resend(db, rows[0].id)
|
||
check("人工重发:回到 pending 并当场叫醒(点重发的人就是想看结果)",
|
||
resent is not None and resent.status == models.NotifyDelivery.STATUS_PENDING
|
||
and len(WAKES) == 1,
|
||
(getattr(resent, "status", None), WAKES))
|
||
if resent is not None:
|
||
park(db, [resent])
|
||
|
||
# ---------------------------------------------- 7. 叫醒只改"何时醒",不改跑什么
|
||
from app import main as main_module
|
||
|
||
stir, stop = main_module._scheduler_stir, main_module._scheduler_stop
|
||
saved_stop = stop.is_set()
|
||
stop.clear()
|
||
stir.clear()
|
||
started = time.monotonic()
|
||
main_module._wake_scheduler()
|
||
check("被叫醒时 _scheduler_sleep 立刻返回,且不退出线程",
|
||
main_module._scheduler_sleep(30) is False and time.monotonic() - started < 5,
|
||
f"等了 {time.monotonic() - started:.2f}s")
|
||
stop.set()
|
||
stir.clear()
|
||
started = time.monotonic()
|
||
main_module._wake_scheduler()
|
||
check("退出信号优先:睡里被叫醒也要认出该退出",
|
||
main_module._scheduler_sleep(30) is True and time.monotonic() - started < 5,
|
||
f"等了 {time.monotonic() - started:.2f}s")
|
||
if saved_stop:
|
||
stop.set()
|
||
else:
|
||
stop.clear()
|
||
stir.clear()
|
||
|
||
# ---------------------------------------------- 8. 接线:调度线程真的在听这一层
|
||
# 上面那几条验的是"入队 → 该叫",这一段验"叫了 → 线程真的醒":
|
||
# ``start_scheduler`` 里那句登记把 ``_wake_scheduler`` 接到本层。这里把 Thread 换成
|
||
# 不起动的假类,只验接线,不在校验进程里真跑一轮调度(那会去抢租约、去发 parked 的行)。
|
||
real_thread = main_module.threading.Thread
|
||
|
||
class QuietThread(real_thread):
|
||
def __init__(self, *args, **kwargs): # 记下参数,但不起线程
|
||
self.args, self.kwargs = args, kwargs
|
||
|
||
def start(self):
|
||
return None
|
||
|
||
main_module.threading.Thread = QuietThread
|
||
try:
|
||
main_module.start_scheduler()
|
||
finally:
|
||
main_module.threading.Thread = real_thread
|
||
check("start_scheduler 把调度线程登记成叫醒对象(渠道与事件共享写回那条通路)",
|
||
main_module._wake_scheduler in wake._hooks, len(wake._hooks))
|
||
main_module._scheduler_stir.clear()
|
||
wake.now()
|
||
check("叫醒通路 → 调度线程那条睡真的被中断", main_module._scheduler_stir.is_set())
|
||
main_module._scheduler_stir.clear()
|
||
|
||
|
||
def main() -> int:
|
||
db = SessionLocal()
|
||
try:
|
||
_run(db)
|
||
finally:
|
||
cleanup(db)
|
||
print()
|
||
if FAILS:
|
||
print(f"{len(FAILS)} 条没对上:")
|
||
for item in FAILS:
|
||
print(f" - {item}")
|
||
else:
|
||
print(f"全部 {PASSED} 条对上。")
|
||
return 1 if FAILS else 0
|
||
|
||
|
||
def cleanup(db) -> None:
|
||
db.execute(delete(models.NotifyDelivery).where(or_(
|
||
models.NotifyDelivery.channel == CH,
|
||
models.NotifyDelivery.title.like(f"{TITLE}%"))))
|
||
db.execute(delete(models.OpenEvent).where(or_(models.OpenEvent.webhook == HOOK,
|
||
models.OpenEvent.job == JOB)))
|
||
db.execute(delete(models.OpenWebhook).where(models.OpenWebhook.code == HOOK))
|
||
db.execute(delete(models.NotifyChannel).where(models.NotifyChannel.code == CH))
|
||
db.commit()
|
||
print("[清理] 渠道、订阅、投递记录与事件已删干净")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|