312 lines
16 KiB
Python
312 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""取数通道上「地址带参数」这件事的验收:不打真上游,内存库。
|
||
|
||
PYTHONPATH=. python3 scripts/check_fetch_url_args.py
|
||
|
||
钉的是 2026-10 那次「报销按 OA 单号查详情的接口配不进取数源」:对面给的是
|
||
|
||
GET https://reimb.example.com/api/reimbursements/oa/by-flow-number?oa_flow_number=FYBX202610260014
|
||
|
||
单号是**每张单都不一样**的值,而它写在地址上、请求体是空的。这一趟钉五件事:
|
||
|
||
1. **地址模板真的被替换**,且换进去的每个值都按 URL 编码(值里一个 ``&`` 不能凭空
|
||
多带出一个参数,中文与空格不能拼出一句非法 URL);写死的地址一个字都不动。
|
||
2. **``{arg.x}`` 与请求体同源**:同一个入参表,写在地址上和写在请求体里读到的是同一个值。
|
||
3. **入参空着不外呼**:只看请求体会让空单号照样打出去,白留一条 error 还推高熔断。
|
||
4. **缓存键要认地址**:``?no={work:serial}`` 这一类不落在入参表里,不进键就是
|
||
全公司的单据共用同一份缓存——第二张单拿到第一张单的值,界面上一点动静都没有。
|
||
5. **候选搜索的关键词要真的上线**(2026-11 生产:SCC 商机那一格下拉打字没反应):
|
||
运行时给的 ``keyword`` 只有一个去处——模板里引用它的那几个字。地址上写了
|
||
``?keyword={arg.keyword}`` 就每次跟着变,一个字都没写就每次打同一句话,
|
||
而界面上那一格明明配了一把搜索框,看上去像"搜索坏了",其实是这一格没接。
|
||
|
||
只改本进程内存里的那条源与 ``fetch._cache``,不写库(库是内存的,跑完就没了)。
|
||
共享缓存那一层(Redis)在这一趟里被摘掉:开发机上那台 Redis 会把上一次跑剩下的值喂回来,
|
||
TTL 之内这一趟测的就是"上一次的缓存"而不是这次的行为;那一层自己由 ``check_shared_cache.py`` 管。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
||
|
||
from sqlalchemy import create_engine, event, select # noqa: E402
|
||
from sqlalchemy.orm import Session # noqa: E402
|
||
from sqlalchemy.pool import StaticPool # noqa: E402
|
||
|
||
from app import cache as shared_cache, models # noqa: E402
|
||
from app.service import binding, fetch # noqa: E402
|
||
|
||
PASSED = 0
|
||
FAILS: list[str] = []
|
||
#: 假上游收到的那几笔 ``(方法, 地址)``
|
||
SENT: list[tuple[str, str]] = []
|
||
#: 假上游按地址里的单号回的那点余额
|
||
BALANCES = {"FYBX202610260014": 1200.5, "FYBX202610260015": 88.0}
|
||
|
||
|
||
def eq(name: str, got, want) -> None:
|
||
global PASSED
|
||
if got == want:
|
||
PASSED += 1
|
||
print(f"[通过] {name}")
|
||
else:
|
||
FAILS.append(name)
|
||
print(f"[FAIL] {name} -> 期望 {want!r},实得 {got!r}")
|
||
|
||
|
||
def ok(name: str, condition, detail=None) -> None:
|
||
global PASSED
|
||
if condition:
|
||
PASSED += 1
|
||
print(f"[通过] {name}")
|
||
else:
|
||
FAILS.append(name)
|
||
print(f"[FAIL] {name} -> {detail if detail is not None else '条件不成立'}")
|
||
|
||
|
||
def new_session() -> Session:
|
||
"""内存库。模型里 MySQL 专属的 server_default 在 sqlite 上建不出来,建表前摘掉。"""
|
||
for table in models.Base.metadata.tables.values():
|
||
for column in table.columns:
|
||
if column.server_default is not None:
|
||
column.server_default = None
|
||
engine = create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})
|
||
|
||
@event.listens_for(engine, "before_cursor_execute")
|
||
def _reset_rollback(conn, cursor, statement, parameters, context, executemany): # noqa: ANN001
|
||
# 服务层一处都不显式 rollback,内存库上留着事务会把后面的读挡住
|
||
pass
|
||
|
||
models.Base.metadata.create_all(engine)
|
||
return Session(engine)
|
||
|
||
|
||
def source(db: Session, *, code: str, url: str, body: dict | None = None,
|
||
method: str = "GET", cache_seconds: int = 0) -> models.DataSource:
|
||
row = models.DataSource(
|
||
id=code, code=code, name=f"取数地址参数校验·{code}", system="REIMB",
|
||
kind=models.DataSource.KIND_FETCH, provider=models.DataSource.PROVIDER_HTTP,
|
||
enabled=True, method=method, url=url, headers={}, body_template=body or {},
|
||
pick="data.balance", auth_type=models.DataSource.AUTH_NONE,
|
||
cache_seconds=cache_seconds, timeout_seconds=5, retries=0,
|
||
)
|
||
db.add(row)
|
||
db.commit()
|
||
return row
|
||
|
||
|
||
class Work:
|
||
"""假单据:地址上写 ``{work:serial}`` 时读的就是这一行的 ``serial``。"""
|
||
|
||
def __init__(self, serial: str) -> None:
|
||
self.id = f"work-{serial}"
|
||
self.serial = serial
|
||
self.job = f"job-{serial}"
|
||
|
||
|
||
def fake_http(method: str, url: str, body, headers, timeout): # noqa: ANN001
|
||
SENT.append((str(method), str(url)))
|
||
number = url.rsplit("=", 1)[-1]
|
||
balance = BALANCES.get(number)
|
||
text = '{"code":0,"data":{"balance":%s}}' % ("null" if balance is None else balance)
|
||
return (200, text)
|
||
|
||
|
||
def search_source(db: Session, *, code: str, url: str) -> models.DataSource:
|
||
"""列表型(候选搜索)那一条源:列表节点是 ``data``,值取 ``code``、显示名取 ``name``。"""
|
||
row = source(db, code=code, url=url)
|
||
row.kind = models.DataSource.KIND_SEARCH
|
||
row.items_path = "data"
|
||
row.pick = ""
|
||
row.map_value = "code"
|
||
row.map_label = "name"
|
||
db.commit()
|
||
return row
|
||
|
||
|
||
#: 假上游那两张商机,按关键词筛
|
||
PROJECTS = [("V002462", "兴业银行重庆分行新机房"), ("V002498", "大连市第三人民医院续采")]
|
||
|
||
|
||
def fake_search(method: str, url: str, body, headers, timeout): # noqa: ANN001
|
||
"""按地址上的 ``keyword`` 筛一轮,好让"关键词有没有上线"看得见的假上游。"""
|
||
SENT.append((str(method), str(url)))
|
||
word = urllib.parse.parse_qs(urllib.parse.urlparse(url).query).get("keyword", [""])[0]
|
||
rows = [{"code": code, "name": name} for code, name in PROJECTS if not word or word in name]
|
||
return 200, json.dumps({"code": 0, "data": rows}, ensure_ascii=False)
|
||
|
||
|
||
def main() -> int:
|
||
db = new_session()
|
||
fetch._cache.clear()
|
||
real_http = fetch.http_request
|
||
real_get_json, real_set_json = shared_cache.get_json, shared_cache.set_json
|
||
# 摘掉共享缓存:只留本进程那一份 L1,这一趟测的是"这次的行为"
|
||
shared_cache.get_json = lambda *args, **kwargs: None # type: ignore[assignment]
|
||
shared_cache.set_json = lambda *args, **kwargs: None # type: ignore[assignment]
|
||
fetch.http_request = fake_http # type: ignore[assignment]
|
||
try:
|
||
# ---------------- 1. 地址模板被替换,值按 URL 编码 ----------------
|
||
tpl = ("https://reimb.invalid/api/reimbursements/oa/by-flow-number"
|
||
"?oa_flow_number={arg.processNo}")
|
||
row = source(db, code="url_tpl", url=tpl)
|
||
SENT.clear()
|
||
result = fetch.run(db, row, {"processNo": "FYBX202610260014"}, check_permission=False)
|
||
eq("按单号查的接口真打了出去", [item[1] for item in SENT],
|
||
["https://reimb.invalid/api/reimbursements/oa/by-flow-number"
|
||
"?oa_flow_number=FYBX202610260014"])
|
||
eq("取回来的值是那一格的余额", result.value, 1200.5)
|
||
|
||
row.url = "https://reimb.invalid/api/x?kw={arg.kw}"
|
||
db.commit()
|
||
SENT.clear()
|
||
fetch.run(db, row, {"kw": "a&b=c#中文 空格"}, check_permission=False)
|
||
eq("值里的 & # 空格与中文都被编码,不会多带出一个参数", SENT,
|
||
[("GET", "https://reimb.invalid/api/x?kw=a%26b%3Dc%23%E4%B8%AD%E6%96%87%20%E7%A9%BA%E6%A0%BC")])
|
||
|
||
row.url = "https://reimb.invalid/api/x/{arg.no}/detail"
|
||
db.commit()
|
||
SENT.clear()
|
||
fetch.run(db, row, {"no": "FYBX202610260014"}, check_permission=False)
|
||
eq("拼进路径那一段是同一条规矩", SENT,
|
||
[("GET", "https://reimb.invalid/api/x/FYBX202610260014/detail")])
|
||
|
||
static = source(db, code="url_static", url="https://reimb.invalid/api/budget/balance")
|
||
SENT.clear()
|
||
fetch.run(db, static, {"processNo": "FYBX202610260014"}, check_permission=False)
|
||
eq("写死的地址一个字都不动(存量行为)", [item[1] for item in SENT],
|
||
["https://reimb.invalid/api/budget/balance"])
|
||
eq("写死的地址不进缓存键(升级前后的键一样,存量缓存不作废)",
|
||
"key" if "__url" not in fetch._key_args({"a": "1"}, url="") else "url", "key")
|
||
|
||
row.url = tpl
|
||
db.commit()
|
||
SENT.clear()
|
||
fetch.run(db, row, {"processNo": "FYBX202610260015"}, check_permission=False)
|
||
eq("{arg.x} 与请求体同源:读的是同一张入参表", [item[1] for item in SENT],
|
||
[tpl.replace("{arg.processNo}", "FYBX202610260015")])
|
||
|
||
# 请求体里也写同一个入参:两份都要换,不能只顾一头
|
||
both = source(db, code="url_and_body",
|
||
url="https://reimb.invalid/api/y?no={arg.processNo}",
|
||
body={"oaFlowNumber": "{arg.processNo}", "day": "{auto:today}"},
|
||
method="POST")
|
||
SENT.clear()
|
||
fetch.run(db, both, {"processNo": "FYBX202610260014"}, check_permission=False)
|
||
eq("地址与请求体同时引用同一个入参时,两边都被替换", SENT[0][1],
|
||
"https://reimb.invalid/api/y?no=FYBX202610260014")
|
||
|
||
# ---------------- 2. 入参空着不外呼 ----------------
|
||
SENT.clear()
|
||
empty = fetch.run(db, row, {"processNo": ""}, check_permission=False)
|
||
eq("单号还没填时一次都不外呼", SENT, [])
|
||
ok("挡下来的话点名的是地址上那个入参", "processNo" in (empty.error or ""), empty.error)
|
||
eq("算作被挡住而不是上游报错(不推高熔断)", empty.health, False)
|
||
eq("地址上的入参也算进「缺哪几个」",
|
||
binding.missing_args(row, {"processNo": ""}), ["processNo"])
|
||
# 列表型源照旧不挡空关键词:候选第一页本来就允许空着搜
|
||
search = source(db, code="url_search",
|
||
url="https://reimb.invalid/api/z?keyword={arg.keyword}")
|
||
row_search = db.get(models.DataSource, "url_search")
|
||
row_search.kind = models.DataSource.KIND_SEARCH
|
||
db.commit()
|
||
SENT.clear()
|
||
fetch.run(db, row_search, {"keyword": ""}, check_permission=False)
|
||
eq("列表型源的空关键词照旧打出去", [item[1] for item in SENT],
|
||
["https://reimb.invalid/api/z?keyword="])
|
||
|
||
# ---------------- 3. 缓存键要认地址 ----------------
|
||
shared = source(db, code="url_cache_work",
|
||
url="https://reimb.invalid/api/c?no={work:serial}", cache_seconds=60)
|
||
fetch._cache.clear()
|
||
SENT.clear()
|
||
first = fetch.run(db, shared, {}, work=Work("FYBX202610260014"), check_permission=False)
|
||
second = fetch.run(db, shared, {}, work=Work("FYBX202610260015"), check_permission=False)
|
||
eq("第一张单取到第一张的余额", first.value, 1200.5)
|
||
eq("第二张单不许拿第一张的缓存(入参表是空的,值只写在地址上)",
|
||
second.value, 88.0)
|
||
third = fetch.run(db, shared, {}, work=Work("FYBX202610260014"), check_permission=False)
|
||
eq("同一张单第二次命中缓存", third.cached, True)
|
||
eq("命中缓存时不再打上游:两张单各打一次", len(SENT), 2)
|
||
|
||
# 写在入参表上的那种,键里本来就有这一项,不该因为地址重复算一遍
|
||
keyed = source(db, code="url_cache_arg", url=tpl, cache_seconds=60)
|
||
fetch._cache.clear()
|
||
a = fetch.run(db, keyed, {"processNo": "FYBX202610260014"}, check_permission=False)
|
||
SENT.clear()
|
||
b = fetch.run(db, keyed, {"processNo": "FYBX202610260014"}, check_permission=False)
|
||
eq("同参数第二次命中缓存(键没被地址搅乱)", [a.value, b.value], [1200.5, 1200.5])
|
||
eq("命中缓存时没有外呼", SENT, [])
|
||
|
||
# ---------------- 4. 候选搜索的关键词到不到得了上游 ----------------
|
||
# 列表型源发出去的入参表里永远有 keyword 这一项(``api/data_sources.py:_search_result``),
|
||
# 但模板只认自己引用过的名字:没写就等于每打一次都问同一句话。
|
||
fetch.http_request = fake_search # type: ignore[assignment]
|
||
hit = search_source(db, code="kw_url",
|
||
url="https://crm.invalid/api/opportunities/projects?keyword={arg.keyword}")
|
||
words = []
|
||
for word in ("", "重庆", "大连"):
|
||
SENT.clear()
|
||
got = fetch.run(db, hit, {"keyword": word}, check_permission=False)
|
||
words.append((SENT[0][1], len(got.items)))
|
||
eq("配了写法:中文关键词按 URL 编码上线,空着就是空参数",
|
||
[item[0] for item in words],
|
||
["https://crm.invalid/api/opportunities/projects?keyword=",
|
||
"https://crm.invalid/api/opportunities/projects?keyword=%E9%87%8D%E5%BA%86",
|
||
"https://crm.invalid/api/opportunities/projects?keyword=%E5%A4%A7%E8%BF%9E"])
|
||
eq("上游按关键词回的那一页真的被换掉了", [item[1] for item in words], [2, 1, 1])
|
||
|
||
miss = search_source(db, code="kw_bare",
|
||
url="https://crm.invalid/api/opportunities/projects")
|
||
urls = []
|
||
for word in ("重庆", "大连"):
|
||
SENT.clear()
|
||
fetch.run(db, miss, {"keyword": word}, check_permission=False)
|
||
urls.append(SENT[0][1])
|
||
eq("一个字都没写时两次打出去的是同一句话(界面上就是「打字没反应」)", urls[0], urls[1])
|
||
# 地址接上关键词之后,"不带任何入参的那一趟"必须照旧打得出去:外部字典同步走的正是
|
||
# 这一条(``oa_dict_type.source_args`` 是空的),把它当"入参没填"挡下来,
|
||
# 字典就再也同步不到东西,表单那一格直接空掉
|
||
SENT.clear()
|
||
blank = fetch.run(db, hit, {}, check_permission=False)
|
||
eq("空入参那一趟照旧外呼(外部字典同步走的就是这一条)",
|
||
[item[1] for item in SENT], ["https://crm.invalid/api/opportunities/projects?keyword="])
|
||
eq("空入参不算缺参", binding.missing_args(hit, {}), [])
|
||
ok("这一趟仍然取得到整份清单", blank.ok and len(blank.items) == len(PROJECTS), blank.error)
|
||
eq("请求体那条通道同理:正文里没写 keyword 就不发(``_body_has_keyword``)",
|
||
fetch._body_has_keyword(miss), False)
|
||
|
||
# 留痕里要看得见"搜的是什么":keyword 曾经被"键名里有 key 就当密钥"那条规则抹掉,
|
||
# 于是排查搜索问题时翻调用留痕只看得到一次空入参的调用(``_looks_secret``)。
|
||
trail = db.execute(
|
||
select(models.DataSourceLog).where(models.DataSourceLog.source == "kw_url")
|
||
).scalars().all()
|
||
eq("调用留痕记下每一趟搜的那句话",
|
||
[row.args["keyword"] for row in trail if "keyword" in (row.args or {})],
|
||
["", "重庆", "大连"])
|
||
eq("真密钥仍然被抹掉", fetch._looks_secret("apiKey"), True)
|
||
finally:
|
||
fetch.http_request = real_http # type: ignore[assignment]
|
||
shared_cache.get_json, shared_cache.set_json = real_get_json, real_set_json
|
||
fetch._cache.clear()
|
||
db.close()
|
||
|
||
print()
|
||
if FAILS:
|
||
print(f"失败 {len(FAILS)} 条:")
|
||
for name in FAILS:
|
||
print(f" - {name}")
|
||
return 1
|
||
print(f"全部通过({PASSED} 条)")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|