unis-oa/backend/scripts/check_fetch_url_args.py

312 lines
16 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
"""取数通道上「地址带参数」这件事的验收:不打真上游,内存库。
PYTHONPATH=. python3 scripts/check_fetch_url_args.py
钉的是 2026-10 那次「报销按 OA 单号查详情的接口配不进取数源」:对面给的是
GET https://reimb.example.com/api/reimbursements/oa/by-flow-number?oa_flow_number=FYBX202610260014
单号是**每张单都不一样**的值,而它写在地址上、请求体是空的。这一趟钉五件事:
1. **地址模板真的被替换**,且换进去的每个值都按 URL 编码(值里一个 ``&`` 不能凭空
多带出一个参数,中文与空格不能拼出一句非法 URL);写死的地址一个字都不动。
2. **``{arg.x}`` 与请求体同源**:同一个入参表,写在地址上和写在请求体里读到的是同一个值。
3. **入参空着不外呼**:只看请求体会让空单号照样打出去,白留一条 error 还推高熔断。
4. **缓存键要认地址**:``?no={work:serial}`` 这一类不落在入参表里,不进键就是
全公司的单据共用同一份缓存——第二张单拿到第一张单的值,界面上一点动静都没有。
5. **候选搜索的关键词要真的上线**(2026-11 生产:SCC 商机那一格下拉打字没反应):
运行时给的 ``keyword`` 只有一个去处——模板里引用它的那几个字。地址上写了
``?keyword={arg.keyword}`` 就每次跟着变,一个字都没写就每次打同一句话,
而界面上那一格明明配了一把搜索框,看上去像"搜索坏了",其实是这一格没接。
只改本进程内存里的那条源与 ``fetch._cache``,不写库(库是内存的,跑完就没了)。
共享缓存那一层(Redis)在这一趟里被摘掉:开发机上那台 Redis 会把上一次跑剩下的值喂回来,
TTL 之内这一趟测的就是"上一次的缓存"而不是这次的行为;那一层自己由 ``check_shared_cache.py`` 管。
"""
from __future__ import annotations
import json
import pathlib
import sys
import time
import urllib.parse
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
from sqlalchemy import create_engine, event, select # noqa: E402
from sqlalchemy.orm import Session # noqa: E402
from sqlalchemy.pool import StaticPool # noqa: E402
from app import cache as shared_cache, models # noqa: E402
from app.service import binding, fetch # noqa: E402
PASSED = 0
FAILS: list[str] = []
#: 假上游收到的那几笔 ``(方法, 地址)``
SENT: list[tuple[str, str]] = []
#: 假上游按地址里的单号回的那点余额
BALANCES = {"FYBX202610260014": 1200.5, "FYBX202610260015": 88.0}
def eq(name: str, got, want) -> None:
global PASSED
if got == want:
PASSED += 1
print(f"[通过] {name}")
else:
FAILS.append(name)
print(f"[FAIL] {name} -> 期望 {want!r},实得 {got!r}")
def ok(name: str, condition, detail=None) -> None:
global PASSED
if condition:
PASSED += 1
print(f"[通过] {name}")
else:
FAILS.append(name)
print(f"[FAIL] {name} -> {detail if detail is not None else '条件不成立'}")
def new_session() -> Session:
"""内存库。模型里 MySQL 专属的 server_default 在 sqlite 上建不出来,建表前摘掉。"""
for table in models.Base.metadata.tables.values():
for column in table.columns:
if column.server_default is not None:
column.server_default = None
engine = create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})
@event.listens_for(engine, "before_cursor_execute")
def _reset_rollback(conn, cursor, statement, parameters, context, executemany): # noqa: ANN001
# 服务层一处都不显式 rollback,内存库上留着事务会把后面的读挡住
pass
models.Base.metadata.create_all(engine)
return Session(engine)
def source(db: Session, *, code: str, url: str, body: dict | None = None,
method: str = "GET", cache_seconds: int = 0) -> models.DataSource:
row = models.DataSource(
id=code, code=code, name=f"取数地址参数校验·{code}", system="REIMB",
kind=models.DataSource.KIND_FETCH, provider=models.DataSource.PROVIDER_HTTP,
enabled=True, method=method, url=url, headers={}, body_template=body or {},
pick="data.balance", auth_type=models.DataSource.AUTH_NONE,
cache_seconds=cache_seconds, timeout_seconds=5, retries=0,
)
db.add(row)
db.commit()
return row
class Work:
"""假单据:地址上写 ``{work:serial}`` 时读的就是这一行的 ``serial``。"""
def __init__(self, serial: str) -> None:
self.id = f"work-{serial}"
self.serial = serial
self.job = f"job-{serial}"
def fake_http(method: str, url: str, body, headers, timeout): # noqa: ANN001
SENT.append((str(method), str(url)))
number = url.rsplit("=", 1)[-1]
balance = BALANCES.get(number)
text = '{"code":0,"data":{"balance":%s}}' % ("null" if balance is None else balance)
return (200, text)
def search_source(db: Session, *, code: str, url: str) -> models.DataSource:
"""列表型(候选搜索)那一条源:列表节点是 ``data``,值取 ``code``、显示名取 ``name``。"""
row = source(db, code=code, url=url)
row.kind = models.DataSource.KIND_SEARCH
row.items_path = "data"
row.pick = ""
row.map_value = "code"
row.map_label = "name"
db.commit()
return row
#: 假上游那两张商机,按关键词筛
PROJECTS = [("V002462", "兴业银行重庆分行新机房"), ("V002498", "大连市第三人民医院续采")]
def fake_search(method: str, url: str, body, headers, timeout): # noqa: ANN001
"""按地址上的 ``keyword`` 筛一轮,好让"关键词有没有上线"看得见的假上游。"""
SENT.append((str(method), str(url)))
word = urllib.parse.parse_qs(urllib.parse.urlparse(url).query).get("keyword", [""])[0]
rows = [{"code": code, "name": name} for code, name in PROJECTS if not word or word in name]
return 200, json.dumps({"code": 0, "data": rows}, ensure_ascii=False)
def main() -> int:
db = new_session()
fetch._cache.clear()
real_http = fetch.http_request
real_get_json, real_set_json = shared_cache.get_json, shared_cache.set_json
# 摘掉共享缓存:只留本进程那一份 L1,这一趟测的是"这次的行为"
shared_cache.get_json = lambda *args, **kwargs: None # type: ignore[assignment]
shared_cache.set_json = lambda *args, **kwargs: None # type: ignore[assignment]
fetch.http_request = fake_http # type: ignore[assignment]
try:
# ---------------- 1. 地址模板被替换,值按 URL 编码 ----------------
tpl = ("https://reimb.invalid/api/reimbursements/oa/by-flow-number"
"?oa_flow_number={arg.processNo}")
row = source(db, code="url_tpl", url=tpl)
SENT.clear()
result = fetch.run(db, row, {"processNo": "FYBX202610260014"}, check_permission=False)
eq("按单号查的接口真打了出去", [item[1] for item in SENT],
["https://reimb.invalid/api/reimbursements/oa/by-flow-number"
"?oa_flow_number=FYBX202610260014"])
eq("取回来的值是那一格的余额", result.value, 1200.5)
row.url = "https://reimb.invalid/api/x?kw={arg.kw}"
db.commit()
SENT.clear()
fetch.run(db, row, {"kw": "a&b=c#中文 空格"}, check_permission=False)
eq("值里的 & # 空格与中文都被编码,不会多带出一个参数", SENT,
[("GET", "https://reimb.invalid/api/x?kw=a%26b%3Dc%23%E4%B8%AD%E6%96%87%20%E7%A9%BA%E6%A0%BC")])
row.url = "https://reimb.invalid/api/x/{arg.no}/detail"
db.commit()
SENT.clear()
fetch.run(db, row, {"no": "FYBX202610260014"}, check_permission=False)
eq("拼进路径那一段是同一条规矩", SENT,
[("GET", "https://reimb.invalid/api/x/FYBX202610260014/detail")])
static = source(db, code="url_static", url="https://reimb.invalid/api/budget/balance")
SENT.clear()
fetch.run(db, static, {"processNo": "FYBX202610260014"}, check_permission=False)
eq("写死的地址一个字都不动(存量行为)", [item[1] for item in SENT],
["https://reimb.invalid/api/budget/balance"])
eq("写死的地址不进缓存键(升级前后的键一样,存量缓存不作废)",
"key" if "__url" not in fetch._key_args({"a": "1"}, url="") else "url", "key")
row.url = tpl
db.commit()
SENT.clear()
fetch.run(db, row, {"processNo": "FYBX202610260015"}, check_permission=False)
eq("{arg.x} 与请求体同源:读的是同一张入参表", [item[1] for item in SENT],
[tpl.replace("{arg.processNo}", "FYBX202610260015")])
# 请求体里也写同一个入参:两份都要换,不能只顾一头
both = source(db, code="url_and_body",
url="https://reimb.invalid/api/y?no={arg.processNo}",
body={"oaFlowNumber": "{arg.processNo}", "day": "{auto:today}"},
method="POST")
SENT.clear()
fetch.run(db, both, {"processNo": "FYBX202610260014"}, check_permission=False)
eq("地址与请求体同时引用同一个入参时,两边都被替换", SENT[0][1],
"https://reimb.invalid/api/y?no=FYBX202610260014")
# ---------------- 2. 入参空着不外呼 ----------------
SENT.clear()
empty = fetch.run(db, row, {"processNo": ""}, check_permission=False)
eq("单号还没填时一次都不外呼", SENT, [])
ok("挡下来的话点名的是地址上那个入参", "processNo" in (empty.error or ""), empty.error)
eq("算作被挡住而不是上游报错(不推高熔断)", empty.health, False)
eq("地址上的入参也算进「缺哪几个」",
binding.missing_args(row, {"processNo": ""}), ["processNo"])
# 列表型源照旧不挡空关键词:候选第一页本来就允许空着搜
search = source(db, code="url_search",
url="https://reimb.invalid/api/z?keyword={arg.keyword}")
row_search = db.get(models.DataSource, "url_search")
row_search.kind = models.DataSource.KIND_SEARCH
db.commit()
SENT.clear()
fetch.run(db, row_search, {"keyword": ""}, check_permission=False)
eq("列表型源的空关键词照旧打出去", [item[1] for item in SENT],
["https://reimb.invalid/api/z?keyword="])
# ---------------- 3. 缓存键要认地址 ----------------
shared = source(db, code="url_cache_work",
url="https://reimb.invalid/api/c?no={work:serial}", cache_seconds=60)
fetch._cache.clear()
SENT.clear()
first = fetch.run(db, shared, {}, work=Work("FYBX202610260014"), check_permission=False)
second = fetch.run(db, shared, {}, work=Work("FYBX202610260015"), check_permission=False)
eq("第一张单取到第一张的余额", first.value, 1200.5)
eq("第二张单不许拿第一张的缓存(入参表是空的,值只写在地址上)",
second.value, 88.0)
third = fetch.run(db, shared, {}, work=Work("FYBX202610260014"), check_permission=False)
eq("同一张单第二次命中缓存", third.cached, True)
eq("命中缓存时不再打上游:两张单各打一次", len(SENT), 2)
# 写在入参表上的那种,键里本来就有这一项,不该因为地址重复算一遍
keyed = source(db, code="url_cache_arg", url=tpl, cache_seconds=60)
fetch._cache.clear()
a = fetch.run(db, keyed, {"processNo": "FYBX202610260014"}, check_permission=False)
SENT.clear()
b = fetch.run(db, keyed, {"processNo": "FYBX202610260014"}, check_permission=False)
eq("同参数第二次命中缓存(键没被地址搅乱)", [a.value, b.value], [1200.5, 1200.5])
eq("命中缓存时没有外呼", SENT, [])
# ---------------- 4. 候选搜索的关键词到不到得了上游 ----------------
# 列表型源发出去的入参表里永远有 keyword 这一项(``api/data_sources.py:_search_result``),
# 但模板只认自己引用过的名字:没写就等于每打一次都问同一句话。
fetch.http_request = fake_search # type: ignore[assignment]
hit = search_source(db, code="kw_url",
url="https://crm.invalid/api/opportunities/projects?keyword={arg.keyword}")
words = []
for word in ("", "重庆", "大连"):
SENT.clear()
got = fetch.run(db, hit, {"keyword": word}, check_permission=False)
words.append((SENT[0][1], len(got.items)))
eq("配了写法:中文关键词按 URL 编码上线,空着就是空参数",
[item[0] for item in words],
["https://crm.invalid/api/opportunities/projects?keyword=",
"https://crm.invalid/api/opportunities/projects?keyword=%E9%87%8D%E5%BA%86",
"https://crm.invalid/api/opportunities/projects?keyword=%E5%A4%A7%E8%BF%9E"])
eq("上游按关键词回的那一页真的被换掉了", [item[1] for item in words], [2, 1, 1])
miss = search_source(db, code="kw_bare",
url="https://crm.invalid/api/opportunities/projects")
urls = []
for word in ("重庆", "大连"):
SENT.clear()
fetch.run(db, miss, {"keyword": word}, check_permission=False)
urls.append(SENT[0][1])
eq("一个字都没写时两次打出去的是同一句话(界面上就是「打字没反应」)", urls[0], urls[1])
# 地址接上关键词之后,"不带任何入参的那一趟"必须照旧打得出去:外部字典同步走的正是
# 这一条(``oa_dict_type.source_args`` 是空的),把它当"入参没填"挡下来,
# 字典就再也同步不到东西,表单那一格直接空掉
SENT.clear()
blank = fetch.run(db, hit, {}, check_permission=False)
eq("空入参那一趟照旧外呼(外部字典同步走的就是这一条)",
[item[1] for item in SENT], ["https://crm.invalid/api/opportunities/projects?keyword="])
eq("空入参不算缺参", binding.missing_args(hit, {}), [])
ok("这一趟仍然取得到整份清单", blank.ok and len(blank.items) == len(PROJECTS), blank.error)
eq("请求体那条通道同理:正文里没写 keyword 就不发(``_body_has_keyword``)",
fetch._body_has_keyword(miss), False)
# 留痕里要看得见"搜的是什么":keyword 曾经被"键名里有 key 就当密钥"那条规则抹掉,
# 于是排查搜索问题时翻调用留痕只看得到一次空入参的调用(``_looks_secret``)。
trail = db.execute(
select(models.DataSourceLog).where(models.DataSourceLog.source == "kw_url")
).scalars().all()
eq("调用留痕记下每一趟搜的那句话",
[row.args["keyword"] for row in trail if "keyword" in (row.args or {})],
["", "重庆", "大连"])
eq("真密钥仍然被抹掉", fetch._looks_secret("apiKey"), True)
finally:
fetch.http_request = real_http # type: ignore[assignment]
shared_cache.get_json, shared_cache.set_json = real_get_json, real_set_json
fetch._cache.clear()
db.close()
print()
if FAILS:
print(f"失败 {len(FAILS)} 条:")
for name in FAILS:
print(f" - {name}")
return 1
print(f"全部通过({PASSED} 条)")
return 0
if __name__ == "__main__":
sys.exit(main())