212 lines
9.9 KiB
Python
212 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
||
"""守 ``app/cache`` 的批量读与字典那一层的预取:省的是往返,不许动口径。
|
||
|
||
``/api/v1/draft/types`` 一次要读八个字典。逐个读是八趟 Redis 往返(本部署到 Redis
|
||
一趟 9ms,实测这一页 138ms 里 105ms 在等 Redis),所以加了两件事:
|
||
:func:`app.cache.get_many_json`(一趟 ``MGET`` 读一批键)与
|
||
:meth:`app.service.dicts.DictCache.prefetch`(按整张表的字典清单一次预取)。
|
||
|
||
这个脚本一次钉四条:
|
||
|
||
1. **批量读与逐个读答案一模一样**:命中的值相同,没命中 / 没这个键 / 装的不是合法
|
||
JSON 的键一律"不在结果里",与 :func:`app.cache.get_json` 回 ``None`` 同口径;
|
||
2. **预取不改变解析结果**:预取过的 :class:`DictCache` 与没预取的逐项给出同一份选项;
|
||
3. **作废的即时性**:改了字典项并 ``invalidate`` 之后,新的一次预取必须立刻是新答案;
|
||
4. **没配 Redis 也照跑**:这一层整个不参与时,上面这些结论仍然成立(退回逐个现查);
|
||
5. **共享缓存没命中时也只问一条 ``IN``**:字典缓存到期或刚被作废那一趟,七个字典从
|
||
逐条七趟(本部署实测 472ms)并成一条(28ms);一条 ``IN`` 分出来的组与逐条现查
|
||
逐字段一样,空的与查无此码的字典不回写共享缓存(否则真建了项要等作废才看得见)。
|
||
|
||
库里不留东西:建的那点临时字典项跑在事务里,最后回滚;共享缓存里那个临时键也删掉。::
|
||
|
||
PYTHONPATH=. python3 scripts/check_shared_cache.py
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sys
|
||
|
||
from sqlalchemy import select
|
||
|
||
from app import cache, models
|
||
from app.db import SessionLocal
|
||
from app.service import dicts
|
||
|
||
FAILS: list[str] = []
|
||
PASSED = 0
|
||
CODE = "zz-check-shared"
|
||
|
||
|
||
def check(label: str, ok: bool, detail: str = "") -> None:
|
||
global PASSED
|
||
if ok:
|
||
PASSED += 1
|
||
print(f"[通过] {label}")
|
||
else:
|
||
FAILS.append(label)
|
||
print(f"[FAIL] {label}{f' -> {detail}' if detail else ''}")
|
||
|
||
|
||
def temp_items(db: Session) -> list[models.DictItem]:
|
||
"""临时字典:两个字典项,跑在事务里,脚本结束整体回滚。"""
|
||
db.execute # noqa: B018 - 只是提醒下面用的是 ORM 而不是裸 SQL
|
||
rows = [
|
||
models.DictItem(id=f"chk-shared-{index}", type=CODE, value=f"v{index}",
|
||
label=f"选项{index}", enabled=True, order_number=index)
|
||
for index in (1, 2)
|
||
]
|
||
db.add_all(rows)
|
||
db.flush()
|
||
return rows
|
||
|
||
|
||
def batch_read(db: Session) -> None:
|
||
"""批量读与逐个读必须同口径。"""
|
||
key_hit = cache.key("check", "shared", "hit")
|
||
key_bad = cache.key("check", "shared", "bad")
|
||
key_gone = cache.key("check", "shared", "gone")
|
||
payload = [{"value": "v1", "label": "选项1"}, {"value": "v2", "label": "选项2"}]
|
||
cache.set_json(key_hit, payload, 60)
|
||
cache.set_value(key_bad, "这不是 JSON", 60)
|
||
cache.delete(key_gone)
|
||
|
||
if not cache.configured():
|
||
check("没配 Redis:批量读整批当没命中(与逐个读回 None 同口径)",
|
||
cache.get_many_json([key_hit, key_bad, key_gone]) == {}
|
||
and cache.get_json(key_hit) is None)
|
||
return
|
||
|
||
found = cache.get_many_json([key_hit, key_bad, key_gone, " ", ""])
|
||
check("批量读:命中的值与逐个读一字不差",
|
||
found.get(key_hit) == cache.get_json(key_hit) == payload, str(found))
|
||
check("批量读:没这个键 / 空名 / 非法 JSON 都不进结果",
|
||
key_gone not in found and "" not in found and key_bad not in found
|
||
and cache.get_json(key_bad) is None, str(sorted(found)))
|
||
check("批量读:重复的键只回一份,值仍对得上",
|
||
cache.get_many_json([key_hit, key_hit, key_hit]) == {key_hit: payload})
|
||
check("批量读:空清单不发命令也不报错", cache.get_many_json([]) == {} and cache.get_many_json([""]) == {})
|
||
cache.delete(key_hit)
|
||
check("删掉的键:批量读立刻读不到(作废口径与逐个读一致)",
|
||
cache.get_many_json([key_hit]) == {} and cache.get_json(key_hit) is None)
|
||
cache.delete(key_bad)
|
||
|
||
|
||
def dict_items(db: Session) -> None:
|
||
"""预取过与没预取,选项必须一模一样;改了字典项当场见得到。"""
|
||
temp_items(db)
|
||
want = dicts.items_of(db, CODE, enabled_only=True)
|
||
check("素材:临时字典读得到两项", [item["value"] for item in want] == ["v1", "v2"], str(want))
|
||
|
||
warm = dicts.DictCache(db)
|
||
warm.prefetch([CODE, "zz.查无此字典", "", None])
|
||
check("预取不改变解析结果:预取过的与没预取的一项不差",
|
||
warm.items(CODE) == dicts.DictCache(db).items(CODE) == want,
|
||
str(warm.items(CODE)))
|
||
check("预取认得出的字典码之外的值不当成字典(查无此字典回空表)",
|
||
warm.items("zz.查无此字典") == [])
|
||
check("已经问过的字典码:再预取一次不换答案",
|
||
(warm.prefetch([CODE]), warm.items(CODE))[1] == want)
|
||
|
||
# 停用一项:作废之后新的一次解析必须当场是新答案
|
||
row = db.execute(
|
||
select(models.DictItem).where(models.DictItem.type == CODE).order_by(models.DictItem.order_number)
|
||
).scalars().first()
|
||
if row is not None:
|
||
row.enabled = False
|
||
db.flush()
|
||
dicts.invalidate(CODE)
|
||
fresh = dicts.DictCache(db)
|
||
fresh.prefetch([CODE])
|
||
left = [item["value"] for item in fresh.items(CODE)]
|
||
check("改完字典项并作废:新的预取当场就少一项(没预取的那份也一样)",
|
||
left == ["v2"] and dicts.DictCache(db).items(CODE) == fresh.items(CODE), str(left))
|
||
row.enabled = True
|
||
db.flush()
|
||
dicts.invalidate(CODE)
|
||
back = dicts.DictCache(db)
|
||
back.prefetch([CODE])
|
||
check("还原:选项数也复原", [item["value"] for item in back.items(CODE)] == ["v1", "v2"])
|
||
|
||
check("字段清单里明细表列上的字典码也一起预取(codes_of 走得到)",
|
||
dicts.codes_of([{"type": "select", "dictCode": "a"},
|
||
{"type": "table", "columns": [{"dictCode": "b"}]}]) == ["a", "b"])
|
||
|
||
|
||
def batch_when_shared_misses(db: Session) -> None:
|
||
"""共享缓存整批没命中时:一条 ``IN`` 取齐,答案与逐条现查逐字段一致。
|
||
|
||
这是字典缓存每 TTL 到期一次、或有人刚改过字典之后的那一趟——批量预取只并
|
||
共享缓存那一半的话,库上还是逐条排队,实测七个字典 472ms。
|
||
"""
|
||
from sqlalchemy import event
|
||
|
||
from app.db import engine as db_engine
|
||
|
||
codes = [CODE, "zz.查无此字典", "zz.空字典"]
|
||
key = dicts._shared_key(CODE, True)
|
||
cache.delete(*[dicts._shared_key(code, True) for code in codes])
|
||
|
||
shared_on = dicts.shared_cache_seconds() > 0
|
||
# 先让共享缓存热起来,再清掉:模拟"TTL 刚到期"而不是"从来没写过"。
|
||
# 字典共享被关掉(dict_cache_seconds = 0)时这一层整个不参与,跳过这一步素材。
|
||
warm = dicts.DictCache(db)
|
||
warm.prefetch([CODE])
|
||
if shared_on:
|
||
check("素材:这一本字典确实进过共享缓存", isinstance(cache.get_json(key), list),
|
||
repr(cache.get_json(key))[:60])
|
||
cache.delete(*[dicts._shared_key(code, True) for code in codes])
|
||
|
||
hits: list[str] = []
|
||
|
||
def _grab(conn, cursor, statement, parameters, context, executemany):
|
||
if "oa_dict_item" in statement:
|
||
hits.append("IN" if " IN (" in statement else "one")
|
||
|
||
event.listen(db_engine, "before_cursor_execute", _grab)
|
||
try:
|
||
cold = dicts.DictCache(db)
|
||
cold.prefetch(codes)
|
||
finally:
|
||
event.remove(db_engine, "before_cursor_execute", _grab)
|
||
|
||
check("整批没命中时只问一条 IN(不再逐条排队)", hits == ["IN"], f"实际问到 {hits}")
|
||
# 对照的那份必须绕过共享缓存现查:_fill_from_db 顺手把结果回写了缓存,
|
||
# 拿 items_of 当对照等于拿"刚被这条路写过的那份"比它自己,顺序错了也照样通过。
|
||
want = [dicts.item_payload(row) for row in dicts.item_rows(db, CODE, enabled_only=True)]
|
||
check("一条 IN 分出来的组与逐条现查逐字段一致(含先后顺序)", cold.items(CODE) == want,
|
||
f"批量 {[i['value'] for i in cold.items(CODE)]} / 逐条 {[i['value'] for i in want]}")
|
||
check("查无此码 / 空字典都回空表,与逐条口径一样",
|
||
cold.items("zz.查无此字典") == [] and cold.items("zz.空字典") == [])
|
||
check("空字典不回写共享缓存(真建了项不必等作废)",
|
||
cache.get_json(dicts._shared_key("zz.空字典", True)) is None, "空表被写进了共享缓存")
|
||
if not shared_on:
|
||
check("字典共享关掉时这一层只并库上那一趟,不写任何键",
|
||
all(cache.get_json(dicts._shared_key(code, True)) is None for code in codes),
|
||
"关着共享还写进了 Redis")
|
||
cache.delete(*[dicts._shared_key(code, True) for code in codes])
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
db = SessionLocal()
|
||
try:
|
||
print(f"共享缓存:{'已配置' if cache.configured() else '没配置(这一层不参与)'}"
|
||
f",客户端{'连上了' if cache.client() is not None else '没有'}")
|
||
batch_read(db)
|
||
dict_items(db)
|
||
# 素材复用上面那两行:temp_items 用固定主键,插两次会撞 PRIMARY
|
||
batch_when_shared_misses(db)
|
||
finally:
|
||
# 缓存里那个临时字典键要删干净:库里那几行回滚了,缓存留着就是个指向不存在字典的键
|
||
cache.delete_prefix(cache.key("dict", CODE))
|
||
db.rollback()
|
||
db.close()
|
||
if FAILS:
|
||
print(f"共 {len(FAILS)} 项不通过:{';'.join(FAILS)}")
|
||
return 1
|
||
print(f"通过:{PASSED} 项(批量读与逐个读同口径、预取不改结果、作废当场生效、冷缓存只问一条 IN)")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|