unis-oa/backend/scripts/check_shared_cache.py

212 lines
9.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
"""守 ``app/cache`` 的批量读与字典那一层的预取:省的是往返,不许动口径。
``/api/v1/draft/types`` 一次要读八个字典。逐个读是八趟 Redis 往返(本部署到 Redis
一趟 9ms,实测这一页 138ms 里 105ms 在等 Redis),所以加了两件事:
:func:`app.cache.get_many_json`(一趟 ``MGET`` 读一批键)与
:meth:`app.service.dicts.DictCache.prefetch`(按整张表的字典清单一次预取)。
这个脚本一次钉四条:
1. **批量读与逐个读答案一模一样**:命中的值相同,没命中 / 没这个键 / 装的不是合法
JSON 的键一律"不在结果里",与 :func:`app.cache.get_json` 回 ``None`` 同口径;
2. **预取不改变解析结果**:预取过的 :class:`DictCache` 与没预取的逐项给出同一份选项;
3. **作废的即时性**:改了字典项并 ``invalidate`` 之后,新的一次预取必须立刻是新答案;
4. **没配 Redis 也照跑**:这一层整个不参与时,上面这些结论仍然成立(退回逐个现查);
5. **共享缓存没命中时也只问一条 ``IN``**:字典缓存到期或刚被作废那一趟,七个字典从
逐条七趟(本部署实测 472ms)并成一条(28ms);一条 ``IN`` 分出来的组与逐条现查
逐字段一样,空的与查无此码的字典不回写共享缓存(否则真建了项要等作废才看得见)。
库里不留东西:建的那点临时字典项跑在事务里,最后回滚;共享缓存里那个临时键也删掉。::
PYTHONPATH=. python3 scripts/check_shared_cache.py
"""
from __future__ import annotations
import sys
from sqlalchemy import select
from app import cache, models
from app.db import SessionLocal
from app.service import dicts
FAILS: list[str] = []
PASSED = 0
CODE = "zz-check-shared"
def check(label: str, ok: bool, detail: str = "") -> None:
global PASSED
if ok:
PASSED += 1
print(f"[通过] {label}")
else:
FAILS.append(label)
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
def temp_items(db: Session) -> list[models.DictItem]:
"""临时字典:两个字典项,跑在事务里,脚本结束整体回滚。"""
db.execute # noqa: B018 - 只是提醒下面用的是 ORM 而不是裸 SQL
rows = [
models.DictItem(id=f"chk-shared-{index}", type=CODE, value=f"v{index}",
label=f"选项{index}", enabled=True, order_number=index)
for index in (1, 2)
]
db.add_all(rows)
db.flush()
return rows
def batch_read(db: Session) -> None:
"""批量读与逐个读必须同口径。"""
key_hit = cache.key("check", "shared", "hit")
key_bad = cache.key("check", "shared", "bad")
key_gone = cache.key("check", "shared", "gone")
payload = [{"value": "v1", "label": "选项1"}, {"value": "v2", "label": "选项2"}]
cache.set_json(key_hit, payload, 60)
cache.set_value(key_bad, "这不是 JSON", 60)
cache.delete(key_gone)
if not cache.configured():
check("没配 Redis:批量读整批当没命中(与逐个读回 None 同口径)",
cache.get_many_json([key_hit, key_bad, key_gone]) == {}
and cache.get_json(key_hit) is None)
return
found = cache.get_many_json([key_hit, key_bad, key_gone, " ", ""])
check("批量读:命中的值与逐个读一字不差",
found.get(key_hit) == cache.get_json(key_hit) == payload, str(found))
check("批量读:没这个键 / 空名 / 非法 JSON 都不进结果",
key_gone not in found and "" not in found and key_bad not in found
and cache.get_json(key_bad) is None, str(sorted(found)))
check("批量读:重复的键只回一份,值仍对得上",
cache.get_many_json([key_hit, key_hit, key_hit]) == {key_hit: payload})
check("批量读:空清单不发命令也不报错", cache.get_many_json([]) == {} and cache.get_many_json([""]) == {})
cache.delete(key_hit)
check("删掉的键:批量读立刻读不到(作废口径与逐个读一致)",
cache.get_many_json([key_hit]) == {} and cache.get_json(key_hit) is None)
cache.delete(key_bad)
def dict_items(db: Session) -> None:
"""预取过与没预取,选项必须一模一样;改了字典项当场见得到。"""
temp_items(db)
want = dicts.items_of(db, CODE, enabled_only=True)
check("素材:临时字典读得到两项", [item["value"] for item in want] == ["v1", "v2"], str(want))
warm = dicts.DictCache(db)
warm.prefetch([CODE, "zz.查无此字典", "", None])
check("预取不改变解析结果:预取过的与没预取的一项不差",
warm.items(CODE) == dicts.DictCache(db).items(CODE) == want,
str(warm.items(CODE)))
check("预取认得出的字典码之外的值不当成字典(查无此字典回空表)",
warm.items("zz.查无此字典") == [])
check("已经问过的字典码:再预取一次不换答案",
(warm.prefetch([CODE]), warm.items(CODE))[1] == want)
# 停用一项:作废之后新的一次解析必须当场是新答案
row = db.execute(
select(models.DictItem).where(models.DictItem.type == CODE).order_by(models.DictItem.order_number)
).scalars().first()
if row is not None:
row.enabled = False
db.flush()
dicts.invalidate(CODE)
fresh = dicts.DictCache(db)
fresh.prefetch([CODE])
left = [item["value"] for item in fresh.items(CODE)]
check("改完字典项并作废:新的预取当场就少一项(没预取的那份也一样)",
left == ["v2"] and dicts.DictCache(db).items(CODE) == fresh.items(CODE), str(left))
row.enabled = True
db.flush()
dicts.invalidate(CODE)
back = dicts.DictCache(db)
back.prefetch([CODE])
check("还原:选项数也复原", [item["value"] for item in back.items(CODE)] == ["v1", "v2"])
check("字段清单里明细表列上的字典码也一起预取(codes_of 走得到)",
dicts.codes_of([{"type": "select", "dictCode": "a"},
{"type": "table", "columns": [{"dictCode": "b"}]}]) == ["a", "b"])
def batch_when_shared_misses(db: Session) -> None:
"""共享缓存整批没命中时:一条 ``IN`` 取齐,答案与逐条现查逐字段一致。
这是字典缓存每 TTL 到期一次、或有人刚改过字典之后的那一趟——批量预取只并
共享缓存那一半的话,库上还是逐条排队,实测七个字典 472ms。
"""
from sqlalchemy import event
from app.db import engine as db_engine
codes = [CODE, "zz.查无此字典", "zz.空字典"]
key = dicts._shared_key(CODE, True)
cache.delete(*[dicts._shared_key(code, True) for code in codes])
shared_on = dicts.shared_cache_seconds() > 0
# 先让共享缓存热起来,再清掉:模拟"TTL 刚到期"而不是"从来没写过"。
# 字典共享被关掉(dict_cache_seconds = 0)时这一层整个不参与,跳过这一步素材。
warm = dicts.DictCache(db)
warm.prefetch([CODE])
if shared_on:
check("素材:这一本字典确实进过共享缓存", isinstance(cache.get_json(key), list),
repr(cache.get_json(key))[:60])
cache.delete(*[dicts._shared_key(code, True) for code in codes])
hits: list[str] = []
def _grab(conn, cursor, statement, parameters, context, executemany):
if "oa_dict_item" in statement:
hits.append("IN" if " IN (" in statement else "one")
event.listen(db_engine, "before_cursor_execute", _grab)
try:
cold = dicts.DictCache(db)
cold.prefetch(codes)
finally:
event.remove(db_engine, "before_cursor_execute", _grab)
check("整批没命中时只问一条 IN(不再逐条排队)", hits == ["IN"], f"实际问到 {hits}")
# 对照的那份必须绕过共享缓存现查:_fill_from_db 顺手把结果回写了缓存,
# 拿 items_of 当对照等于拿"刚被这条路写过的那份"比它自己,顺序错了也照样通过。
want = [dicts.item_payload(row) for row in dicts.item_rows(db, CODE, enabled_only=True)]
check("一条 IN 分出来的组与逐条现查逐字段一致(含先后顺序)", cold.items(CODE) == want,
f"批量 {[i['value'] for i in cold.items(CODE)]} / 逐条 {[i['value'] for i in want]}")
check("查无此码 / 空字典都回空表,与逐条口径一样",
cold.items("zz.查无此字典") == [] and cold.items("zz.空字典") == [])
check("空字典不回写共享缓存(真建了项不必等作废)",
cache.get_json(dicts._shared_key("zz.空字典", True)) is None, "空表被写进了共享缓存")
if not shared_on:
check("字典共享关掉时这一层只并库上那一趟,不写任何键",
all(cache.get_json(dicts._shared_key(code, True)) is None for code in codes),
"关着共享还写进了 Redis")
cache.delete(*[dicts._shared_key(code, True) for code in codes])
def main(argv: list[str]) -> int:
db = SessionLocal()
try:
print(f"共享缓存:{'已配置' if cache.configured() else '没配置(这一层不参与)'}"
f",客户端{'连上了' if cache.client() is not None else '没有'}")
batch_read(db)
dict_items(db)
# 素材复用上面那两行:temp_items 用固定主键,插两次会撞 PRIMARY
batch_when_shared_misses(db)
finally:
# 缓存里那个临时字典键要删干净:库里那几行回滚了,缓存留着就是个指向不存在字典的键
cache.delete_prefix(cache.key("dict", CODE))
db.rollback()
db.close()
if FAILS:
print(f"共 {len(FAILS)} 项不通过:{';'.join(FAILS)}")
return 1
print(f"通过:{PASSED} 项(批量读与逐个读同口径、预取不改结果、作废当场生效、冷缓存只问一条 IN)")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))