2008 lines
103 KiB
Python
2008 lines
103 KiB
Python
"""外部取数执行器 -- 表单字段说「我要 erp_budget_balance 这个源」,怎么调由这里负责。
|
||
|
||
三条不打折的规矩(方案见 ``docs/form-data-integration-design.md`` §5、§8):
|
||
|
||
1. **前端永不直连外部系统**。浏览器直连会死在 CORS、密钥暴露、超时不可控、无审计
|
||
这四件事上,所以调用一律在这个后端代理里发生。
|
||
2. **外部回来的值以这里为准**。``apply_form`` 在提交链路里按同样的入参再取一次并覆盖
|
||
前端传来的值(命中缓存就不重复发请求)——与 ``dicts.apply_auto_fill`` 同一分寸,
|
||
否则「预算余额」这一格本身就能被伪造,审批就成了摆设。
|
||
3. **外部系统只能提供值,不能决定流转**。取回来的值落进 ``work.data`` 之后,条件分支
|
||
与审批人读的仍是 ``work.data``,取数失败按字段上配的 ``onError`` 走,不临场发挥。
|
||
|
||
七件杂事(密钥、超时、重试、缓存、限流、熔断、审计)全在这一层做完,
|
||
字段配置只管点名要哪个源、入参从哪来、值写进哪几格。
|
||
|
||
**「配置说的是什么」不在这一层**:取值写法、多源清单、合并口径、源与引用的校验,
|
||
全在 ``service/binding.py``(候选来源 / 额度 / 阈值 / 流程带数据吃的是同一份,
|
||
分层见 ``docs/data-binding-unification-design.md``)。这一层只负责**执行**。
|
||
|
||
数据源有四条来路:``provider=http`` 打外部系统(``mock`` 是它的样例替身),
|
||
``provider=process`` 查**本平台另一个流程**的单据(执行在 ``service/processdata.py``),
|
||
``provider=sql`` 跑一条管理员配好的只读 SELECT(闸门与执行在 ``service/sqlsource.py``)。
|
||
后者不走网络、没有密钥,但出来的东西照样是一份 ``{"work","data"}`` 形状的"响应",
|
||
所以下面这一整套路——解析、缓存策略、留痕、提交时覆盖、设计期校验——两条来路共用。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import hashlib
|
||
import ipaddress
|
||
import json
|
||
import logging
|
||
import re
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from dataclasses import dataclass, field as dataclass_field
|
||
from decimal import Decimal, InvalidOperation
|
||
from datetime import datetime, timedelta
|
||
from typing import Any, Iterable
|
||
|
||
from sqlalchemy import func, select
|
||
from sqlalchemy.orm import Session
|
||
|
||
from .. import models
|
||
from .. import permissions as perm
|
||
from .. import cache
|
||
from . import binding, presets, processdata, sqlsource
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
#: 字段上挂外部取数配置的属性名
|
||
FIELD_PROP = "fetch"
|
||
#: 明细表那一类字段:列上也能挂 ``fetch``,执行时逐行各跑一次(见 :func:`row_fetch_fields`)
|
||
TABLE_TYPE = "table"
|
||
#: 取数时机里「发起之后冻结」那一档的键名(字段上配 ``fetch.freezeAfterStart``)
|
||
FREEZE_AFTER_START = "freezeAfterStart"
|
||
#: 开始环节的环节类型,与 ``engine.definition.ACTIVITY_BEGIN`` 同一个字面量;
|
||
#: 写死在这里是为了不反过来 import 引擎(引擎那一层正是调用本模块的人)
|
||
BEGIN_TYPE = "begin"
|
||
# 挂几个源、怎么合并、失败策略那几档,口径都在 service/binding.py;
|
||
# 这几个名字留在这里给老调用方与前端镜像注释用,别再往这里加规则。
|
||
MAX_FIELD_SOURCES = binding.MAX_FIELD_SOURCES
|
||
MERGE_MODES = binding.MERGE_MODES
|
||
NUMERIC_MERGE_MODES = binding.NUMERIC_MERGE_MODES
|
||
#: 取数来源与时间留痕;下划线开头的键由服务端自己写,页面回传的那份在
|
||
#: ``engine.without_trace_keys`` 里丢掉,审批轨迹也不把它当人工改动(见 ``form_change_notes``)
|
||
TRACE_KEY = "_fetch"
|
||
#: 单次调用最多几个来回(重试次数由数据源自己配,这里再兜一层上限)
|
||
MAX_RETRIES = 3
|
||
_cache: dict[tuple[str, str], tuple[float, "FetchResult"]] = {}
|
||
CACHE_MAX_ENTRIES = 512
|
||
#: 共享缓存(Redis)里一份结果最大多大。上游回一句 5 MB 的正文时不往 Redis 里塞,
|
||
#: 各进程自己那份 L1 照旧留(L2 是给"多个副本别各打一次上游"用的,不是第二块存储)
|
||
SHARED_MAX_BYTES = 256 * 1024
|
||
|
||
MONEY_TYPES = ("money", "number", "formula")
|
||
|
||
|
||
@dataclass
|
||
class FetchResult:
|
||
"""一次取数的结果。``ok`` 为假时 ``error`` 一定有内容,界面直接显示它。"""
|
||
|
||
ok: bool
|
||
value: Any = None
|
||
extras: dict[str, Any] = dataclass_field(default_factory=dict)
|
||
items: list[dict] = dataclass_field(default_factory=list)
|
||
error: str = ""
|
||
status: int = 0
|
||
ms: int = 0
|
||
cached: bool = False
|
||
raw: str = ""
|
||
#: 一句"这次拿到了,但有个条件该改改"(如 SQL 取数被行数上限截断)。
|
||
#: 刻意不塞进 ``error``:那两个键在界面上一个是绿的、一个是红的,混着用迟早有人
|
||
#: 拿 ``if result.error`` 当失败判据,把一句提醒变成一次误报的取数失败。
|
||
note: str = ""
|
||
#: 这次失败算不算"上游不健康"。入参没填、解析规则取不到值、上游明说业务不通过、
|
||
#: 4xx 这些都是我们这侧或业务侧的事,上游明明是活的,拿它们去拉闸只会连累别人。
|
||
health: bool = True
|
||
|
||
def status_label(self) -> str:
|
||
return models.DataSourceLog.STATUS_OK if self.ok else models.DataSourceLog.STATUS_ERROR
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 注册表读取
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def get_source(db: Session, code: str) -> models.DataSource | None:
|
||
if not code:
|
||
return None
|
||
return db.execute(
|
||
select(models.DataSource).where(models.DataSource.code == str(code).strip())
|
||
).scalar_one_or_none()
|
||
|
||
|
||
def list_sources(db: Session, keyword: str = "", kind: str = "", enabled_only: bool = False) -> list[models.DataSource]:
|
||
stmt = select(models.DataSource)
|
||
text = str(keyword or "").strip()
|
||
if text:
|
||
like = f"%{text}%"
|
||
stmt = stmt.where((models.DataSource.code.like(like)) | (models.DataSource.name.like(like)))
|
||
if kind:
|
||
stmt = stmt.where(models.DataSource.kind == kind)
|
||
if enabled_only:
|
||
stmt = stmt.where(models.DataSource.enabled.is_(True))
|
||
return list(db.execute(stmt.order_by(models.DataSource.system, models.DataSource.code)).scalars().all())
|
||
|
||
|
||
def embed_arg_names(row: models.DataSource) -> set[str]:
|
||
"""一条「可嵌入页面」声明了哪几个入参,也就是那一份**参数白名单**。
|
||
|
||
地址模板模式只写在 ``url`` 上;接口取链接那一档还写在请求体与请求头里,三份合起来
|
||
才是"这个页面吃哪几个参数"(权威判据在 ``service/embed.py:declared_args``,这里不回
|
||
头调它:embed 依赖 fetch,反过来 import 会绕成圈,而这一句只是三份模板求并集)。
|
||
"""
|
||
# 两种来路合起来算,不看 url_mode:模板模式下请求体与请求头本来就是空的,
|
||
# 多算这一份不改变结果,却省下一处"要不要 import embed"的绕圈
|
||
return binding.arg_names_of(row.url, row.headers, row.body_template)
|
||
|
||
|
||
def view(row: models.DataSource) -> dict:
|
||
"""给界面的形状。**密钥只进不出**:只回 ``secret_set``,明文一个字符都不回。"""
|
||
from . import embed as embed_svc # 局部导入:embed 依赖本模块,模块级互相引用会绕成圈
|
||
|
||
return {
|
||
"id": row.id,
|
||
"code": row.code,
|
||
"name": row.name,
|
||
"system": row.system,
|
||
"kind": row.kind,
|
||
"enabled": row.enabled,
|
||
"provider": row.provider or models.DataSource.PROVIDER_HTTP,
|
||
"mockPayload": dict(row.mock_payload or {}),
|
||
"method": row.method,
|
||
"url": row.url,
|
||
"headers": dict(row.headers or {}),
|
||
"bodyTemplate": dict(row.body_template or {}),
|
||
# 参数表格的备注与停用行,界面回显用(发请求时不看)
|
||
"paramMeta": dict(row.param_meta or {}),
|
||
"authType": row.auth_type,
|
||
"authUsername": row.auth_username,
|
||
"secretSet": bool(row.auth_secret),
|
||
"itemsPath": row.items_path,
|
||
"pick": row.pick,
|
||
"mapValue": row.map_value,
|
||
"mapLabel": row.map_label,
|
||
"mapExtra": dict(row.map_extra or {}),
|
||
# provider=process 用得上这两格:取哪个流程、取哪一单(见 service/processdata.py)
|
||
"srcProcess": row.src_process,
|
||
"srcMatch": dict(row.src_match or {}),
|
||
# provider=sql 用的那两条:语句本体 + 它需要哪几个入参。入参清单由后端解析 SQL 给出,
|
||
# 前端不再自己认一遍 :name ——掩码那套逻辑(注释/字符串/反引号)在 TS 里再写一份必漂
|
||
"sqlText": row.sql_text or "",
|
||
"sqlParams": sorted(sqlsource.params(row.sql_text)),
|
||
# provider=embed 用的那两格:免登方式与对方应用编码;地址模板仍走上面的 url
|
||
"pageMode": row.page_mode or "none",
|
||
"appCode": row.app_code or "",
|
||
# provider=embed 那一档"地址由接口现给"的三格:来源、链接在响应里的位置、允许的来源
|
||
"urlMode": embed_svc.url_mode_of(row),
|
||
"urlPath": row.url_path or "",
|
||
# provider=embed:查看页那一条地址。接口档看它在响应里的位置,模板档看第二条地址模板;
|
||
# 这一句"这条源到底给不给得出查看页"是后端算的,界面不必再认一遍哪一档读哪一列
|
||
"viewUrlPath": row.view_url_path or "",
|
||
"viewUrl": row.view_url or "",
|
||
"viewLinkReady": embed_svc.view_link_ready(row),
|
||
# 「还有哪几条链接」那张表与「顺手带回的那几个值」那一栏:界面回显用。另外附一份
|
||
# 按当下这一档筛过的「现在真能给出那几条」——写歪而没存进来的行不该被算成一条
|
||
"urlLinks": embed_svc.link_rows(row),
|
||
"urlMeta": embed_svc.link_meta_map(row),
|
||
# 属性面板那栏下拉直接从这一份长出来:能挑哪几条 = 管理员登记了哪几条
|
||
"links": embed_svc.link_brief(row),
|
||
"urlOrigins": list(row.url_origins or []),
|
||
# provider=embed:「这一页被放弃时通知对方」往哪儿发、怎么发、对方那个单号字段叫什么
|
||
"abandonUrl": row.abandon_url or "",
|
||
"abandonSource": row.abandon_source or "",
|
||
"abandonMethod": row.abandon_method or "",
|
||
"abandonField": row.abandon_field or "",
|
||
"abandonExpr": row.abandon_expr or "",
|
||
"abandonBody": dict(row.abandon_body or {}),
|
||
# 这个页面要哪几个参数:表单属性面板拿它对齐"这一格要带哪些值"。
|
||
# 不只数地址模板——"先调接口拿链接"那一档的参数还写在请求体与请求头里
|
||
"embedArgs": sorted(embed_arg_names(row)),
|
||
"timeoutSeconds": row.timeout_seconds,
|
||
"retries": row.retries,
|
||
"cacheSeconds": row.cache_seconds,
|
||
"rateLimitQpm": row.rate_limit_qpm,
|
||
"cooldownSeconds": row.cooldown_seconds,
|
||
"minPermission": row.min_permission,
|
||
# 出站(审批结果写回)用的几个判定口径,见 docs/outbound-integration-design.md §3.1
|
||
"direction": row.direction or models.DataSource.DIRECTION_IN,
|
||
"successPath": row.success_path,
|
||
"successValue": row.success_value,
|
||
"messagePath": row.message_path,
|
||
"refPath": row.ref_path,
|
||
"shadow": bool(row.shadow),
|
||
"failing": int(row.fail_count or 0),
|
||
"openUntil": row.open_until.strftime("%Y-%m-%d %H:%M:%S") if row.open_until else "",
|
||
"lastTestStatus": row.last_test_status,
|
||
"lastTestMessage": row.last_test_message,
|
||
"lastTestMs": row.last_test_ms,
|
||
"lastTestAt": row.last_test_at.strftime("%Y-%m-%d %H:%M:%S") if row.last_test_at else "",
|
||
"lastTestBy": row.last_test_by,
|
||
"description": row.description,
|
||
"creatorName": row.creator_name,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 请求模板与响应取值
|
||
# ---------------------------------------------------------------------------
|
||
|
||
#: 一段写成 ``字段=值`` 就是"按字段筛行":``expense_items[category_name=差旅费]``。
|
||
#: 也可以比大小(``[amount>5000]`` / ``[日期>=2026-10-01]`` / ``[remark!=打车]``),
|
||
#: 也可以只认开头或只认包含(``[category_name^=差旅费]`` / ``[description~=机票]``):
|
||
#: 明细列表里"取等于某个值的那一行"、"取超过某个数的那一行"与"把差旅那三行算成一笔"
|
||
#: 是同一件事的几种写法,分几套语法只会让人先去翻文档。键名里真带 ``=``(``{"a=b": 1}``)
|
||
#: 不受影响——筛选只在脚下是一份列表时生效,是对象时照旧当键名查。
|
||
FILTER_SEGMENT = re.compile(r"^([^=<>!^~]+?)\s*(!=|\^=|~=|>=|<=|=|>|<)\s*(.*)$", re.S)
|
||
#: 大小比较用的运算符(``=`` / ``!=`` 走 :func:`_matches`,``^=`` / ``~=``, 走文本判据,
|
||
#: 这一份只管比大小)
|
||
COMPARISONS = {"<": lambda a, b: a < b, "<=": lambda a, b: a <= b,
|
||
">": lambda a, b: a > b, ">=": lambda a, b: a >= b}
|
||
#: 一段写成 ``sum(列)`` 就是"把筛剩下的那些行算成一个数":
|
||
#: ``data.expense_items[category_name^=差旅费].sum(amount)``
|
||
AGGREGATE_SEGMENT = re.compile(r"^(sum|avg|max|min|count)\(([^()]*)\)$", re.I)
|
||
#: 筛不到行时,这一列实际有哪几个值最多列这么几个(列全是另一种噪音)
|
||
MISS_VALUES = 5
|
||
|
||
|
||
def pick_steps(path: str) -> list[str]:
|
||
"""把点路径切成一段段:``a.b[0].c`` → ``["a", "b", "0", "c"]``。
|
||
|
||
**中括号里面不切点**:``明细[amount=5643.84]`` 与 ``明细[日期=2026-10-26]`` 里的点
|
||
属于那个值本身,照点号切开就成了 ``amount=5643`` 加一段孤零零的 ``84``——金额与日期
|
||
恰恰是最常被拿去筛的两类值,所以这里不能用一句 ``split(".")`` 打发。
|
||
括号没闭合时整段尾巴当键名的一部分,宁可得不到值也别猜错一段。
|
||
"""
|
||
text = str(path or "").strip()
|
||
out: list[str] = []
|
||
buf = ""
|
||
index = 0
|
||
while index < len(text):
|
||
char = text[index]
|
||
if char == ".":
|
||
if buf:
|
||
out.append(buf)
|
||
buf = ""
|
||
index += 1
|
||
elif char == "[":
|
||
if buf:
|
||
out.append(buf)
|
||
buf = ""
|
||
end = text.find("]", index)
|
||
if end < 0:
|
||
buf = text[index:]
|
||
break
|
||
out.append(text[index + 1:end])
|
||
index = end + 1
|
||
if index < len(text) and text[index] == ".":
|
||
index += 1
|
||
else:
|
||
buf += char
|
||
index += 1
|
||
if buf:
|
||
out.append(buf)
|
||
return out
|
||
|
||
|
||
def _matches(got: Any, want: str) -> bool:
|
||
"""筛行时两边怎么算相等。
|
||
|
||
上游把金额回成数字、把 ``2`` 回成字符串都常见,而配置的人只会照屏幕上看到的那串去写,
|
||
所以 ``amount=5643.84`` 与 ``count=2`` 都得中对的值:两边都能读成数字就按数字比,
|
||
否则去掉首尾空格按文本比。键不存在与值是 ``null`` 都算空,``remark=`` 筛得到空行。
|
||
"""
|
||
target = str(want).strip()
|
||
if got is None or isinstance(got, (dict, list)):
|
||
return target in ("", "null")
|
||
left = str(got).strip()
|
||
if left == target:
|
||
return True
|
||
try:
|
||
return float(left) == float(target)
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
def parse_filter(segment: str) -> tuple[str, str, str] | None:
|
||
"""一段路径是不是筛行写法:``amount>5000`` → ``("amount", ">", "5000")``,不是回 ``None``。
|
||
|
||
切段器与界面(``utils/dsPreset.ts:pickFilter``)共用这一条判据,两边各写一份迟早会漂。
|
||
"""
|
||
hit = FILTER_SEGMENT.match(str(segment))
|
||
if not hit or not hit.group(1).strip():
|
||
return None
|
||
return hit.group(1).strip(), hit.group(2), hit.group(3)
|
||
|
||
|
||
def _compare(got: Any, op: str, want: str) -> bool:
|
||
"""一个单元格按 ``op`` 算不算命中。
|
||
|
||
``=`` / ``!=`` 用的就是相等那条判据(含"空值算空串"那一套),所以 ``remark!=``
|
||
筛的是非空行。``^=`` 认开头、``~=`` 认包含,这是给"差旅费-住宿费 / 差旅费-出差补贴
|
||
这种同族类目要算成一笔"用的,按文本判,``null`` 与取不到的键一律不算命中。
|
||
``>`` / ``>=`` / ``<`` / ``<=`` 两边都能读成数字就按数字比——
|
||
``"5643.84"`` 与 ``5643.84`` 必须是同一个数;读不成数字就按文本比,
|
||
``2026-10-01`` 这一类定长日期串正好吃字典序。取不到键与 ``null`` 不参与比大小。
|
||
"""
|
||
if op == "=":
|
||
return _matches(got, want)
|
||
if op == "!=":
|
||
return not _matches(got, want)
|
||
if op in ("^=", "~="):
|
||
if got is None or isinstance(got, (dict, list)):
|
||
return False
|
||
left = str(got).strip()
|
||
target = str(want).strip()
|
||
return left.startswith(target) if op == "^=" else target in left
|
||
if got is None or isinstance(got, (dict, list)):
|
||
return False
|
||
left, target = str(got).strip(), str(want).strip()
|
||
try:
|
||
pair = (float(left), float(target))
|
||
except ValueError:
|
||
pair = (left, target)
|
||
return COMPARISONS[op](pair[0], pair[1])
|
||
|
||
|
||
def _filter_rows(rows: list, field: str, op: str = "=", want: str = "") -> list:
|
||
"""按 ``字段`` ``op`` ``值`` 筛列表里的对象行,回**筛剩的那些**(不是第一条)。"""
|
||
return [item for item in rows
|
||
if isinstance(item, dict) and _compare(item.get(field), op, want)]
|
||
|
||
|
||
def parse_aggregate(segment: str) -> tuple[str, str] | None:
|
||
"""一段路径是不是聚合写法:``sum(amount)`` → ``("sum", "amount")``,不是回 ``None``。
|
||
|
||
函数名认大小写无所谓,列名可以省(``count()`` 数行数)。界面那边
|
||
(``utils/dsPreset.ts:pickAggregate``)有同样一份判据。
|
||
"""
|
||
hit = AGGREGATE_SEGMENT.match(str(segment).strip())
|
||
if not hit:
|
||
return None
|
||
return hit.group(1).lower(), hit.group(2).strip()
|
||
|
||
|
||
def _number(value: Any) -> Decimal | None:
|
||
"""一个单元格读成的数;读不成数就算它不在这笔合计里。
|
||
|
||
``null``、空串、文本、布尔都不算数。带千分位的 ``"1,200.00"`` 算——上游给金额
|
||
写成字符串时多半就带逗号。用 :class:`~decimal.Decimal` 加,别让三行金额
|
||
加成 ``5763.8399999999995`` 摆在表单上。
|
||
"""
|
||
if value is None or isinstance(value, bool):
|
||
return None
|
||
if isinstance(value, (int, float, Decimal)):
|
||
return Decimal(str(value))
|
||
text = str(value).strip().replace(",", "")
|
||
if not text:
|
||
return None
|
||
try:
|
||
return Decimal(text)
|
||
except InvalidOperation:
|
||
return None
|
||
|
||
|
||
def _aggregate(rows: list, func: str, column: str) -> Any:
|
||
"""把筛剩下的那些行算成一个数。取不到就回 ``None``,由 :func:`pick_miss` 说原因。
|
||
|
||
一行都没筛到时 ``sum`` 回的是 ``None`` 而不是 0:类目名写错的人看见 0 会以为
|
||
"这单就是 0 元",看见报错才知道自己筛空了。``count`` 是唯一可以说没有列名的
|
||
(``count()`` 数行数,空列表就是 0 行,这句话是真的),``sum(列)`` 不写列名时
|
||
合的是列表里那些裸值本身(``data.amounts.sum()``)。
|
||
"""
|
||
if func == "count":
|
||
if not column:
|
||
return len(rows)
|
||
return sum(1 for row in rows
|
||
if isinstance(row, dict) and row.get(column) not in (None, ""))
|
||
if not rows:
|
||
return None
|
||
values = []
|
||
for row in rows:
|
||
cell = row.get(column) if isinstance(row, dict) else (row if not column else None)
|
||
number = _number(cell)
|
||
if number is not None:
|
||
values.append(number)
|
||
if not values:
|
||
return None
|
||
if func == "sum":
|
||
total = sum(values)
|
||
elif func == "avg":
|
||
total = sum(values) / Decimal(len(values))
|
||
else:
|
||
total = max(values) if func == "max" else min(values)
|
||
return float(total)
|
||
|
||
|
||
def pick_path(payload: Any, path: str) -> Any:
|
||
"""按点路径从响应里取一个值;取不到回 ``None``,不抛异常。
|
||
|
||
四种段:对象键(``data.rows``)、数组下标(``rows.0`` 或 ``rows[0]``)、按字段筛行
|
||
(``expense_items[category_name=差旅费]``,含 ``^=`` 认开头 / ``~=`` 认包含)、
|
||
以及把剩下的行算成一个数(``.sum(amount)`` / ``.count()``)。筛完还是一份列表,
|
||
所以**末尾停在筛选上给的就是筛剩的那些行**(列表节点正需要这个),
|
||
后面再接列名时不写下标就取第一条匹配行(``[k=v].amount``);
|
||
要第 2、3 条就自己写 ``[k=v][1].amount``。没筛过而直接给列表点列名仍算配错,
|
||
宁可乐呵呵回一句取不到,也不悄悄拿第一行糊过去。
|
||
"""
|
||
current = payload
|
||
picked = False # 上一步是不是筛选:只有筛出来的列表允许不写下标就取头一条
|
||
for segment in pick_steps(path):
|
||
agg = parse_aggregate(segment)
|
||
if agg and isinstance(current, list):
|
||
current = _aggregate(current, agg[0], agg[1])
|
||
picked = False
|
||
continue
|
||
hit = parse_filter(segment)
|
||
if hit and isinstance(current, list):
|
||
current = _filter_rows(current, *hit)
|
||
picked = True
|
||
continue
|
||
if isinstance(current, dict):
|
||
if segment not in current:
|
||
return None
|
||
current = current[segment]
|
||
elif isinstance(current, list):
|
||
if segment.isdigit():
|
||
if int(segment) >= len(current):
|
||
return None
|
||
current = current[int(segment)]
|
||
elif picked and current and isinstance(current[0], dict) and segment in current[0]:
|
||
current = current[0][segment]
|
||
else:
|
||
return None
|
||
else:
|
||
return None
|
||
picked = False
|
||
return current
|
||
|
||
|
||
def _walked(steps: list[str], index: int) -> str:
|
||
"""前 ``index + 1`` 段拼回一句能照着改的路径:下标与筛选挂中括号,键名才用点。"""
|
||
out = ""
|
||
for segment in steps[: index + 1]:
|
||
hit = parse_filter(segment)
|
||
if hit:
|
||
out += f"[{hit[0]}{hit[1]}{hit[2]}]"
|
||
elif segment.isdigit():
|
||
out += f"[{segment}]"
|
||
elif not out:
|
||
out = segment
|
||
else:
|
||
out += f".{segment}"
|
||
return out
|
||
|
||
|
||
def pick_miss(payload: Any, path: str) -> str:
|
||
"""取不到值时说清卡在第几段;说不上来就回空串,由调用方用那句兜底的。
|
||
|
||
这一句不是装饰:列表型响应里"哪一段断了"与"那一列其实有哪几个值"恰好就是配
|
||
``[字段=值]`` 时唯一想知道的两件事,而界面此刻手上就有一份真实响应,
|
||
不趁这一刻说,人就得回填单页面撞一次才知道自己写错了值。
|
||
"""
|
||
steps = pick_steps(path)
|
||
if not steps:
|
||
return ""
|
||
current = payload
|
||
picked = False
|
||
for index, segment in enumerate(steps):
|
||
hit = parse_filter(segment)
|
||
walked = _walked(steps, index)
|
||
agg = parse_aggregate(segment)
|
||
if agg:
|
||
func, column = agg
|
||
if not isinstance(current, list):
|
||
return f"{walked}:{func}() 只能算在一份列表上,这里不是列表"
|
||
if func == "count":
|
||
current = _aggregate(current, func, column)
|
||
continue
|
||
if not current:
|
||
return f"{walked}:这里一行都没有,{func} 不出数来"
|
||
if not any(_number(row.get(column) if isinstance(row, dict) else row) is not None
|
||
for row in current):
|
||
named = f"「{column}」这一列" if column else "这些值"
|
||
return f"{walked}:筛到的 {len(current)} 行里{named}没有能加起来的数"
|
||
current = _aggregate(current, func, column)
|
||
continue
|
||
if hit and isinstance(current, list):
|
||
field, op, want = hit
|
||
left = _filter_rows(current, field, op, want)
|
||
if left:
|
||
current, picked = left, True
|
||
continue
|
||
# 分不清"这一列全是空值"与"根本没有这一列",人就会照着错的那句去改键名
|
||
present = list(dict.fromkeys(
|
||
str(item.get(field)) for item in current if isinstance(item, dict) and field in item
|
||
))
|
||
if present:
|
||
# 空串与 null 换成人话再去重,否则回出来是"现有的是(空)、(空)"
|
||
shown = list(dict.fromkeys(
|
||
"(空)" if value in ("", "None") else _brief(value, 40) for value in present
|
||
))
|
||
tail = f"现有的是{'、'.join(shown[:MISS_VALUES])}"
|
||
else:
|
||
tail = f"这些行里根本没有「{field}」这一列"
|
||
return f"{walked}:按 {field}{op}{want} 没筛到行,{tail}"
|
||
if isinstance(current, dict):
|
||
if segment in current:
|
||
current, picked = current[segment], False
|
||
continue
|
||
return f"{walked}:响应里没有这一段(上一层的键是 {'、'.join(list(current)[:MISS_VALUES])})"
|
||
if isinstance(current, list):
|
||
if segment.isdigit():
|
||
if int(segment) < len(current):
|
||
current, picked = current[int(segment)], False
|
||
continue
|
||
return f"{walked}:这里只有 {len(current)} 行,取不到第 {int(segment)} 行"
|
||
if picked and current and isinstance(current[0], dict) and segment in current[0]:
|
||
current, picked = current[0][segment], False
|
||
continue
|
||
head = current[0] if current else None
|
||
if isinstance(head, dict):
|
||
return f"{walked}:这里是一份列表,要先写 [0] 或 [字段=值] 挑一行(行里有的键:{'、'.join(list(head)[:MISS_VALUES])})"
|
||
return f"{walked}:这里是一份列表,不是对象"
|
||
return f"{walked}:走到这里已经不是一个对象或列表,没法再往下取"
|
||
return ""
|
||
|
||
|
||
#: ``{arg.x}`` / ``{arg:x}``:取本次调用的入参(写法定义在 ``service/binding.py``)
|
||
ARG_REF = binding.ARG_REF
|
||
|
||
|
||
def render(db: Session, template: Any, data: dict, work=None, actor=None) -> Any:
|
||
"""把请求体模板里的取值写法换成真值;口径在 ``service/binding.render``。
|
||
|
||
``data`` 在这里是**本次调用的入参**(由字段上的 ``args`` 从表单映射而来),
|
||
所以模板里写 ``{arg.costCenter}`` 取的就是那个入参。名字留在这里是给
|
||
字典同步(``dicts.sync``)等老调用方用的,别处新代码直接用 ``binding.render``。
|
||
"""
|
||
return binding.render(db, template, data, work=work, actor=actor)
|
||
|
||
|
||
#: 地址模板里换进去的单个值最多留多少个字符。一条 GET 的地址要短:上游那台网关的
|
||
#: 请求行有上限,调用留痕也只是一列字符串,而"把整段备注塞进查询串"恰好是配的人
|
||
#: 手滑时最容易干的事。同一条线在 ``service/embed.py:MAX_ARG_CHARS``。
|
||
URL_ARG_MAX_CHARS = 500
|
||
|
||
|
||
def _url_arg_text(value: Any) -> str:
|
||
"""换进地址里的那个值该长成**一句话**。分寸与 ``service/embed.py:_as_text`` 一致。
|
||
|
||
两处不能照抄 ``str()``:布尔要跟 JSON 与请求体那一份走(``true`` / ``false``,
|
||
``str(True)`` 是 ``True``,对面 Go 那个 ``*bool`` 接不住);整数值的浮点要去掉小数点
|
||
(``work.data`` 里金额一律是 float,``2.0`` 拼进地址,对面声明成 int 的字段直接 400)。
|
||
"""
|
||
if value is None:
|
||
return ""
|
||
if isinstance(value, bool):
|
||
return "true" if value else "false"
|
||
if isinstance(value, float):
|
||
return str(int(value)) if value.is_integer() else str(value)
|
||
if isinstance(value, (dict, list)):
|
||
return json.dumps(value, ensure_ascii=False)
|
||
return str(value)
|
||
|
||
|
||
def render_url(db: Session, template: Any, args: dict, work=None, actor=None) -> str:
|
||
"""地址模板 → 真正打出去的那条地址。
|
||
|
||
写法与请求体**同源**(``{arg.x}`` / ``{form.x}`` / ``{auto:today}`` / ``{work:serial}``,
|
||
口径在 ``service/binding.py``),多的只有一件事:**换进去的每个值都按 URL 编码**。
|
||
为什么非要有这一层:上游给取数的接口常常就是这种形状——
|
||
|
||
GET https://reimb.example.com/api/reimbursements/oa/by-flow-number?oa_flow_number={arg.processNo}
|
||
|
||
参数写在地址上、请求体是空的。不编码,值里一个 ``&`` 就凭空多带出一个参数、
|
||
一个 ``#`` 把后面半截吃成锚点、一个中文或空格就是一句非法 URL;编码之后拼进
|
||
**路径**那一段(``/by-flow-number/{arg.processNo}``)也是同一条规矩,不用再配第二套。
|
||
认不出的花括号(``{常量}`` 那种)原样留着,与请求体一样:不重造模板语言。
|
||
"""
|
||
text = str(template or "")
|
||
if not text:
|
||
return ""
|
||
# ``{arg.x}`` 在模板里等价于 ``{form.x}``(读的都是本次入参),先归一成后者再走同一份解析
|
||
normalized = binding.ARG_REF.sub(lambda match: "{form:%s}" % match.group(1), text)
|
||
|
||
def replace(match: re.Match[str]) -> str:
|
||
value = binding.resolve(db, match.group(0), args or {}, work=work, actor=actor)
|
||
return urllib.parse.quote(_url_arg_text(value)[:URL_ARG_MAX_CHARS], safe="")
|
||
|
||
return binding.EMBED_REF.sub(replace, normalized)
|
||
|
||
|
||
def auth_headers(row: models.DataSource) -> dict[str, str]:
|
||
"""按鉴权配置拼请求头。密钥从库里读,永远不经过接口回显。"""
|
||
if row.auth_type == models.DataSource.AUTH_API_KEY:
|
||
header = str(row.auth_username or "X-API-Key").strip() or "X-API-Key"
|
||
return {header: str(row.auth_secret or "")}
|
||
if row.auth_type == models.DataSource.AUTH_BEARER:
|
||
return {"Authorization": f"Bearer {row.auth_secret}"} if row.auth_secret else {}
|
||
if row.auth_type == models.DataSource.AUTH_BASIC:
|
||
raw = f"{row.auth_username}:{row.auth_secret}".encode("utf-8")
|
||
return {"Authorization": "Basic " + base64.b64encode(raw).decode("ascii")}
|
||
return {}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 执行
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _key_args(args: dict, pick: str = "", extras: dict | None = None, url: str = "") -> dict:
|
||
"""缓存键用的那份指纹入参。
|
||
|
||
解析规则也要进键:同一张源同一份入参,A 格要 ``data.balance``、B 格要 ``data.ownerName``,
|
||
响应虽是同一份,**解析出来的值不是同一份**。共用一个键就等于 A 格先取,B 格拿余额当名字。
|
||
多出来的这一格只进指纹,不进真正发出去的入参(``payload_args`` 那份没动)。
|
||
|
||
``url`` 同理,只在**地址模板真的被换过**时才传进来(写死的地址前后一样,调用方给空串):
|
||
地址上写 ``?no={work:serial}`` 这一类不落在入参表里,不进键就是全公司的单据共用一份缓存。
|
||
"""
|
||
merged = dict(args or {})
|
||
shape = {"pick": str(pick or "").strip()}
|
||
if extras:
|
||
shape["extras"] = {str(name): str(path) for name, path in extras.items()}
|
||
if len(shape) > 1 or shape["pick"]:
|
||
merged["__shape"] = json.dumps(shape, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||
if url:
|
||
merged["__url"] = url
|
||
return merged
|
||
|
||
|
||
def _args_key(code: str, args: dict) -> str:
|
||
payload = json.dumps(args or {}, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||
return hashlib.sha1(payload.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _cache_get(key: tuple[str, str]) -> FetchResult | None:
|
||
hit = _cache.get(key)
|
||
if hit is not None:
|
||
expires, result = hit
|
||
if expires >= time.time():
|
||
return FetchResult(
|
||
ok=result.ok, value=result.value, extras=dict(result.extras), items=list(result.items),
|
||
error=result.error, status=result.status, ms=result.ms, cached=True, raw=result.raw,
|
||
)
|
||
_cache.pop(key, None)
|
||
# L1 没有再看共享那份:别的副本刚取过的数,本副本不必再打一次上游
|
||
shared = cache.get_json(_shared_key(key))
|
||
if isinstance(shared, dict):
|
||
return _from_shared(shared)
|
||
return None
|
||
|
||
|
||
def _cache_put(key: tuple[str, str], seconds: int, result: FetchResult) -> None:
|
||
if seconds <= 0 or not result.ok:
|
||
return
|
||
if len(_cache) >= CACHE_MAX_ENTRIES: # 简单封顶,别让它无限长
|
||
for stale in list(_cache)[: max(1, len(_cache) // 4)]:
|
||
_cache.pop(stale, None)
|
||
_cache[(key[0], key[1])] = (time.time() + seconds, result)
|
||
_put_shared(key, seconds, result)
|
||
|
||
|
||
def _shared_key(key: tuple[str, str]) -> str:
|
||
"""共享缓存的键:数据源编码 + 入参指纹。改了配置靠 :func:`clear_cache` 主动丢,
|
||
不往键里塞配置版本号——那会让改一次配置就把所有已缓存的值作废。"""
|
||
return cache.key("fetch", key[0], key[1])
|
||
|
||
|
||
def _to_shared(result: FetchResult) -> dict:
|
||
"""只放取回来的**值**本身。``ms`` / ``status`` 是上一次真实调用的过程量,
|
||
``raw`` 是上游响应原文:前者会让界面显示成"这一格用了 0 ms",后者可能带
|
||
个人信息与对方系统的内部字段,都只留在本进程的 L1 里,不进共享缓存。"""
|
||
return {
|
||
"ok": bool(result.ok), "value": result.value, "extras": dict(result.extras),
|
||
"items": list(result.items), "error": result.error, "status": int(result.status),
|
||
}
|
||
|
||
|
||
def _from_shared(payload: dict) -> FetchResult:
|
||
return FetchResult(
|
||
ok=bool(payload.get("ok")), value=payload.get("value"),
|
||
extras=dict(payload.get("extras") or {}), items=list(payload.get("items") or []),
|
||
error=str(payload.get("error") or ""), status=int(payload.get("status") or 0),
|
||
cached=True,
|
||
)
|
||
|
||
|
||
def _put_shared(key: tuple[str, str], seconds: int, result: FetchResult) -> None:
|
||
payload = _to_shared(result)
|
||
try:
|
||
size = len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"))
|
||
except (TypeError, ValueError):
|
||
return
|
||
if size > SHARED_MAX_BYTES:
|
||
logger.debug("取数结果 %s 有 %d 字节,超过共享缓存单值上限,只留本进程这份", key[0], size)
|
||
return
|
||
cache.set_json(_shared_key(key), payload, seconds)
|
||
|
||
|
||
def _drop_local(code: str = "") -> None:
|
||
for key in [key for key in _cache if not code or key[0] == code]:
|
||
_cache.pop(key, None)
|
||
|
||
|
||
def clear_cache(code: str = "") -> None:
|
||
"""改了数据源配置就把它那份缓存丢掉,免得界面还在显示改之前取回的数。
|
||
|
||
三层一起丢:本进程的 L1、Redis 里共享的那一份、以及**别的进程的 L1**(发一条广播)。
|
||
没有 Redis 时就只剩第一层 + 第三层各自到期,行为与接 Redis 之前一致。
|
||
"""
|
||
_drop_local(code)
|
||
cache.delete_prefix(cache.key("fetch", code) if code else cache.key("fetch"))
|
||
cache.publish(cache.TOPIC_FETCH, code)
|
||
|
||
|
||
def _on_shared_clear(code: Any = None) -> None:
|
||
"""收到"别的进程改了数据源":把本进程那份 L1 丢掉(幂等,可能连着收到自己发的那条)。"""
|
||
_drop_local(str(code or ""))
|
||
|
||
|
||
cache.watch(cache.TOPIC_FETCH, _on_shared_clear)
|
||
|
||
|
||
#: 环境代理(开发机上的 Clash 一类)眼里的 127.0.0.1 是它自己那一侧:本机上没人
|
||
#: 监听时它回一句空体的 502,把「假上游没起来」这件事盖得死死的;代理自己没开时,
|
||
#: 本地调用又整句变成「外部系统连不上」。所以本机与内网地址一律直连,
|
||
#: 其余地址照旧尊重 http_proxy 与 no_proxy。
|
||
def is_private_url(url: object) -> bool:
|
||
"""URL 的宿主在公网上路由不到:回环、私网段、运营商级 NAT、``*.localhost`` / ``*.local``。
|
||
|
||
两处要用**同一个**判据,所以写在这一层(``service/embed.py`` 依赖本模块,反过来 import
|
||
会绕成圈):一是环境代理那一环(:class:`_LocalDirectProxyHandler`),二是嵌页面那格
|
||
https 的例外(``service/embed.py:url_problem``)。判据按"公网路由得到吗"划界而不是
|
||
只认回环:联调时目标服务常常挂 ``http://192.168.1.20:8000``,TUN 模式的代理会连它一起
|
||
吞进去,那时的报错又是一句没头没尾的「外部系统连不上」。
|
||
"""
|
||
host = str(urllib.parse.urlsplit(str(url or "")).hostname or "").strip().rstrip(".").lower()
|
||
if not host:
|
||
return False
|
||
if host == "localhost" or host.endswith((".localhost", ".local")):
|
||
return True
|
||
try:
|
||
return not ipaddress.ip_address(host).is_global
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
class _LocalDirectProxyHandler(urllib.request.ProxyHandler):
|
||
"""本机与内网地址绕开环境代理,其余交给父类判(含 ``no_proxy``)。"""
|
||
|
||
def proxy_open(self, request, proxy, proxy_type): # noqa: A002 签名跟着 stdlib
|
||
if is_private_url(request.full_url):
|
||
return None # 回 None = 这一环不接管,后面的 HTTPHandler 直连
|
||
return super().proxy_open(request, proxy, proxy_type)
|
||
|
||
|
||
#: 一份 opener 用到底:build_opener 见到 ProxyHandler 的子类会把默认那个顶掉,
|
||
#: 环境代理因此只在装配时读一次,不在每次取数时重读。
|
||
_opener = urllib.request.build_opener(_LocalDirectProxyHandler())
|
||
|
||
|
||
def http_request_full(
|
||
method: str, url: str, body: bytes | None, headers: dict, timeout: float,
|
||
*, read_limit: int | None = None,
|
||
) -> tuple[int, dict[str, str], str, str]:
|
||
"""发一次请求,把**响应头**与最终地址也带回来(``http_request`` 是它的切片)。
|
||
|
||
要响应头的是嵌页面那一格:能不能被嵌写在 ``X-Frame-Options`` 与 CSP 的
|
||
``frame-ancestors`` 里(见 ``service/embed.py``);要最终地址的是为了看出"302 去了登录页"。
|
||
两条都走同一个 opener,所以"本机地址绕开环境代理"那点讲究只写在这一处。
|
||
"""
|
||
request = urllib.request.Request(url, data=body, headers=headers, method=str(method).upper())
|
||
try:
|
||
with _opener.open(request, timeout=timeout) as response: # noqa: S310
|
||
raw = response.read() if not read_limit else response.read(read_limit)
|
||
headers_out = {str(key).lower(): str(value) for key, value in response.headers.items()}
|
||
return int(response.status or 200), headers_out, raw.decode("utf-8", "replace"), \
|
||
str(response.geturl() or url)
|
||
except urllib.error.HTTPError as exc:
|
||
raw = exc.read() if not read_limit else exc.read(read_limit)
|
||
headers_out = {str(key).lower(): str(value) for key, value in (exc.headers or {}).items()}
|
||
return int(exc.code), headers_out, raw.decode("utf-8", "replace"), str(url)
|
||
|
||
|
||
def http_request(method: str, url: str, body: bytes | None, headers: dict, timeout: float) -> tuple[int, str]:
|
||
_status, _headers, text, _final = http_request_full(method, url, body, headers, timeout)
|
||
return _status, text
|
||
|
||
|
||
def _blocked_reason(db: Session, row: models.DataSource) -> str:
|
||
"""熔断:刚连续失败过,这一段时间之内一次都不该打出去(限流见 :func:`_rate_limit_reason`)。"""
|
||
if row.open_until is not None and row.open_until > datetime.now():
|
||
return f"{row.name} 刚连续失败,{row.open_until.strftime('%H:%M:%S')} 之前先不再外呼"
|
||
return ""
|
||
|
||
|
||
def _rate_limit_reason(db: Session, row: models.DataSource) -> str:
|
||
"""每分钟限额。两处讲究,都是这份代码改过的地方:
|
||
|
||
1. **只在真要打出去之前判**(缓存查过之后)。命中缓存的那一次根本不碰上游,
|
||
不该吃上游的额度;老写法在查缓存之前先数留痕表,而被挡住的调用又会往留痕表里
|
||
补一行 ``cached=false``,等于自己把这一分钟永远占满。
|
||
2. **计数放 Redis**:多副本部署时各进程自己数会把限额放大副本数倍,共享计数器
|
||
才是"这个源每分钟 30 次"的本意,顺带省下每次请求在留痕表上那一次 ``COUNT(*)``。
|
||
Redis 没参与就退回原来那句 COUNT,口径与之前一字不差。
|
||
"""
|
||
limit = int(row.rate_limit_qpm or 0)
|
||
if limit <= 0:
|
||
return ""
|
||
def calls_last_minute() -> int:
|
||
"""接 Redis 之前的老口径:数留痕表里最近一分钟的真实调用(不含命中缓存的)。"""
|
||
return int(db.execute(
|
||
select(func.count(models.DataSourceLog.id)).where(
|
||
models.DataSourceLog.source == row.code,
|
||
models.DataSourceLog.cached.is_(False),
|
||
models.DataSourceLog.created_at >= datetime.now() - timedelta(minutes=1),
|
||
)
|
||
).scalar_one() or 0)
|
||
|
||
allowed, retry_after = cache.limit_check(f"ds:{row.code}", limit, calls_last_minute)
|
||
if allowed:
|
||
return ""
|
||
return (f"{row.name} 每分钟最多 {limit} 次,这一分钟已经用完了"
|
||
+ (f"(约 {retry_after} 秒后再试)" if retry_after else ""))
|
||
|
||
|
||
def _mark_result(db: Session, row: models.DataSource, ok: bool, health: bool = True) -> None:
|
||
if ok:
|
||
row.fail_count = 0
|
||
row.open_until = None
|
||
return
|
||
if not health:
|
||
# 拉闸要挡的是"打过去没人接",不是"这单填得不全":后者计入熔断,
|
||
# 一张没选项目的单子就能把整张源对所有人生生关掉几十秒,
|
||
# 而且填对了项目的人还会被报成「刚连续失败」,查的人根本想不到是从哪儿开始的。
|
||
return
|
||
row.fail_count = int(row.fail_count or 0) + 1
|
||
# 一次抖动就静默一分钟会把填单页卡得太狠;连错两次才拉闸
|
||
if row.fail_count >= 2 and int(row.cooldown_seconds or 0) > 0:
|
||
row.open_until = datetime.now() + timedelta(seconds=int(row.cooldown_seconds))
|
||
row.fail_count = 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 给"不是取数、但同样要打外部"的那条路用的两道闸
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def outbound_guard(db: Session, row: models.DataSource) -> str:
|
||
"""真要往外部打一次之前先问一句:熔断还开着吗、这一分钟的额度用完没。挡住了回一句人话。
|
||
|
||
:func:`run` 之外也有要打外部的路——嵌页面那一格「先调接口拿链接」就是
|
||
(见 ``service/embed.py:resolve_link``)。限流与熔断护的是**上游**,不是取数这一条
|
||
通道专属:审批页上那个刷新按钮点快了,同样该被挡住,不然一张源能被一个页面打穿。
|
||
这里只回判据、不留痕——留痕交给调用方,它才知道自己这一趟算哪一类调用。
|
||
"""
|
||
return _blocked_reason(db, row) or _rate_limit_reason(db, row)
|
||
|
||
|
||
def outbound_mark(db: Session, row: models.DataSource, ok: bool, *, health: bool = True) -> None:
|
||
"""把刚打完的那一次记进熔断那本账(与 :func:`run` 记的是同一本)。
|
||
|
||
``health=False`` 留给"是我们这侧的事"那一部分:解析规则取不到值、链接来源没配全,
|
||
上游明明答话了,为它拉闸只会连累别人(与 :func:`run` 里那句注释同一个分寸)。
|
||
"""
|
||
_mark_result(db, row, ok, health)
|
||
|
||
|
||
def _log(
|
||
db: Session,
|
||
row: models.DataSource,
|
||
*,
|
||
args: dict,
|
||
status: str,
|
||
ms: int,
|
||
cached: bool = False,
|
||
http_status: int = 0,
|
||
message: str = "",
|
||
actor=None,
|
||
work=None,
|
||
activity=None,
|
||
) -> None:
|
||
db.add(
|
||
models.DataSourceLog(
|
||
source=row.code,
|
||
kind=row.kind,
|
||
work=str(getattr(work, "id", "") or ""),
|
||
job=str(getattr(work, "job", "") or ""),
|
||
activity=str(getattr(activity, "id", "") or ""),
|
||
actor=str(getattr(actor, "person_id", "") or ""),
|
||
actor_name=str(getattr(actor, "person_name", "") or getattr(actor, "display_name", "") or ""),
|
||
args={key: value for key, value in (args or {}).items() if not _looks_secret(key)},
|
||
status=status,
|
||
http_status=int(http_status or 0),
|
||
ms=int(ms or 0),
|
||
cached=bool(cached),
|
||
message=str(message or "")[:512],
|
||
)
|
||
)
|
||
|
||
|
||
def _looks_secret(key: object) -> bool:
|
||
text = str(key or "").lower()
|
||
if text == "keyword":
|
||
# 候选搜索那一格传的是"人在下拉里打了什么",不是凭证。当年这么写是为了把 apiKey
|
||
# 那一类键挡掉,顺带把 keyword 也当密钥抹了——留痕里因此永远看不到谁搜了什么,
|
||
# 而排查"这个下拉为什么搜不到"恰好要看这一句(2026-11 生产上就是这么绕的)。
|
||
return False
|
||
return any(mark in text for mark in ("password", "secret", "token", "key", "authorization"))
|
||
|
||
|
||
def run(
|
||
db: Session,
|
||
row: models.DataSource,
|
||
args: dict | None = None,
|
||
*,
|
||
actor=None,
|
||
work=None,
|
||
activity=None,
|
||
raw: bool = False,
|
||
check_permission: bool = True,
|
||
pick: str = "",
|
||
extras: dict | None = None,
|
||
bypass_cache: bool = False,
|
||
) -> FetchResult:
|
||
"""调一个数据源。失败不抛异常,回一个 ``ok=False`` 的结果由调用方决定挡还是放。
|
||
|
||
``bypass_cache`` 只跳过"读缓存"那一步(仍会写),给页面上手动重新取数用。
|
||
|
||
``pick`` / ``extras`` 是调用方(表单上那一格)对**解析规则**的覆盖,不改上游调用本身:
|
||
同一条接口,这一格要 ``data.balance``、那一格要 ``data.ownerName``,不用为每种用法
|
||
另建一条源。留空就是跟随源上的声明,行为与以前一个字不差。
|
||
|
||
|
||
顺序固定:**权限 -> 缓存 -> 入参齐不齐 -> 熔断 -> 限流 -> 真调**,前五道都在本地,
|
||
挡住的情况一次 HTTP 都不发(所以留痕里记的是 ``blocked`` 而不是 ``error``)。熔断与
|
||
限流排在缓存后面是有意的:缓存里现成的值不欠上游一次调用,更不该因为"上游刚被打爆"
|
||
就让人看不到已经取回来的数。入参齐不齐排在熔断前面也是有意的——依赖的格子还没填,
|
||
这一格要的是"等一等"这句话,不是别人留下的那句「刚连续失败」。
|
||
|
||
``check_permission=False`` 只给"系统替全员调一次"的场合用(外部字典同步:
|
||
见 ``service/dicts.sync``)。留痕照记调用人,权限那一道交给上层的设计期授权,
|
||
别拿它当跳过审计的开关。
|
||
"""
|
||
payload_args = dict(args or {})
|
||
if check_permission and row.min_permission and actor is not None and not _may_call(db, actor, row.min_permission):
|
||
result = FetchResult(ok=False, error=f"调用 {row.name} 需要「{row.min_permission}」权限")
|
||
_log(db, row, args=payload_args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0,
|
||
message=result.error, actor=actor, work=work, activity=activity)
|
||
return result
|
||
|
||
provider = row.provider or models.DataSource.PROVIDER_HTTP
|
||
if provider == models.DataSource.PROVIDER_PROCESS:
|
||
# 查本平台自己的库:熔断、限流、缓存这三件为"别把外部系统打爆"而生的事在这里没有对手,
|
||
# 所以整段绕开它们,只留权限那一道(上面已经过了)与留痕
|
||
return _run_process(db, row, payload_args, raw=raw, actor=actor, work=work,
|
||
activity=activity, pick=pick, extras=extras)
|
||
if provider == models.DataSource.PROVIDER_MOCK:
|
||
return _run_mock(db, row, payload_args, raw=raw, actor=actor, work=work,
|
||
activity=activity, pick=pick, extras=extras)
|
||
|
||
# 地址先渲染,再算缓存键:写死地址的源换前换后一模一样,键不动(存量缓存照旧命中);
|
||
# 地址上带取值写法的那一类要拿渲染出来的地址进键,否则不落在入参表里的写法
|
||
# (``{work:serial}`` / ``{auto:today}``)会让所有单据共用同一份缓存。
|
||
url = render_url(db, row.url, payload_args, work=work, actor=actor)
|
||
key = (row.code, _args_key(row.code, _key_args(
|
||
payload_args, pick, extras, url=url if url != str(row.url or "") else "")))
|
||
# ``bypass_cache`` 是给填单页那颗「重新取数」用的:人当场要的就是"现在这一秒的值",
|
||
# 拿缓存里那份回来等于没刷。取回来仍然写回缓存,下一趟照常命中。
|
||
if not raw and not bypass_cache:
|
||
hit = _cache_get(key)
|
||
if hit is not None:
|
||
_log(db, row, args=payload_args, status=hit.status_label(), ms=hit.ms, cached=True,
|
||
http_status=hit.status, actor=actor, work=work, activity=activity)
|
||
return hit
|
||
|
||
missing = binding.missing_args(row, payload_args)
|
||
if missing:
|
||
# 请求体或地址里写着要用这几个入参,人却还没填:打过去也是一句业务错误回来,
|
||
# 白占一次额度、白留一条 error、还顺手把熔断往上推。就地说明白是谁没填。
|
||
names = "、".join(missing)
|
||
reason = f"取数入参 {names} 还是空的,这一格要等它对应的那一格先填上,本次没有外呼"
|
||
_log(db, row, args=payload_args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0,
|
||
message=reason, actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=reason, health=False)
|
||
|
||
reason = _blocked_reason(db, row) or _rate_limit_reason(db, row)
|
||
if reason:
|
||
result = FetchResult(ok=False, error=reason)
|
||
_log(db, row, args=payload_args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0,
|
||
message=reason, actor=actor, work=work, activity=activity)
|
||
return result
|
||
|
||
if provider == models.DataSource.PROVIDER_SQL:
|
||
# 查本库的一条只读查询:没有密钥、没有网络,但缓存、限流、留痕、解析一样不少
|
||
return _run_sql(db, row, key, payload_args, raw=raw, actor=actor, work=work,
|
||
activity=activity, pick=pick, extras=extras)
|
||
|
||
if row.shadow and row.direction == models.DataSource.DIRECTION_OUT:
|
||
# 影子模式的口径只有一句:**这张源不打真请求**。发件箱那侧(outbox.send)照这句办,
|
||
# 设计页的「测试连通」走的也是这一条路,不在这儿挡住,点一下测试就在生产上游
|
||
# 落了一张真单据——而"先配着、上游还没就绪"恰恰是开影子的典型场景。
|
||
reason = "这张源开了影子模式,本次没有外呼。要真打一次,先到「写回判定」关掉影子模式"
|
||
_log(db, row, args=payload_args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0,
|
||
message=reason, actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=reason, health=False)
|
||
|
||
body = render(db, row.body_template or {}, payload_args, work=work, actor=actor)
|
||
if row.kind == models.DataSource.KIND_SEARCH and _body_has_keyword(row):
|
||
body.setdefault("keyword", str(payload_args.get("keyword") or ""))
|
||
headers = {"Content-Type": "application/json", **{str(k): str(v) for k, v in (row.headers or {}).items()},
|
||
**auth_headers(row)}
|
||
data = json.dumps(body, ensure_ascii=False).encode("utf-8") if body else None
|
||
timeout = max(1, int(row.timeout_seconds or 5))
|
||
attempts = max(1, min(MAX_RETRIES, int(row.retries or 0) + 1))
|
||
|
||
started = time.perf_counter()
|
||
status, text = 0, ""
|
||
last_error = ""
|
||
for attempt in range(attempts):
|
||
try:
|
||
status, text = http_request(row.method, url, data, headers, timeout)
|
||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||
status, last_error = 0, friendly_error(exc)
|
||
if attempt + 1 < attempts:
|
||
time.sleep(0.2 * (attempt + 1))
|
||
continue
|
||
_finish_fail(db, row, payload_args, started, last_error, actor, work, activity)
|
||
return FetchResult(ok=False, error=last_error, ms=_ms(started))
|
||
if 200 <= status < 300:
|
||
break
|
||
detail = _brief(text)
|
||
last_error = f"上游返回 {status}:{detail}" if detail else f"上游返回 {status},响应体是空的"
|
||
# 4xx 是配置或参数错了,重试没有意义;5xx 才值得再试一次
|
||
if status < 500 or attempt + 1 >= attempts:
|
||
# 4xx 是这边把参数或配置写错了,上游好得很,不该为它拉闸;5xx 才算它病了
|
||
_finish_fail(db, row, payload_args, started, last_error, actor, work, activity,
|
||
http_status=status, health=status >= 500)
|
||
return FetchResult(ok=False, error=last_error, status=status, ms=_ms(started))
|
||
time.sleep(0.2 * (attempt + 1))
|
||
|
||
ms = _ms(started)
|
||
try:
|
||
parsed = json.loads(text or "{}")
|
||
except ValueError:
|
||
message = f"响应不是 JSON:{_brief(text)}"
|
||
_finish_fail(db, row, payload_args, started, message, actor, work, activity, http_status=status)
|
||
return FetchResult(ok=False, error=message, status=status, ms=ms)
|
||
|
||
result = parse(row, parsed, pick=pick, extras=extras)
|
||
result.status, result.ms, result.raw = status, ms, text if raw else ""
|
||
_mark_result(db, row, result.ok, result.health)
|
||
_log(db, row, args=payload_args, status=result.status_label(), ms=ms, http_status=status,
|
||
message=result.error, actor=actor, work=work, activity=activity)
|
||
if result.ok:
|
||
_cache_put(key, int(row.cache_seconds or 0), result)
|
||
return result
|
||
|
||
|
||
def _run_mock(
|
||
db: Session, row: models.DataSource, args: dict, *,
|
||
raw: bool = False, actor=None, work=None, activity=None,
|
||
pick: str = "", extras: dict | None = None
|
||
) -> FetchResult:
|
||
"""样例通道:不打外部,把配好的样例响应按**同一套解析规则**走一遍。
|
||
|
||
它存在的理由很实在:上游系统还没开发时,解析规则、字段绑定、额度阈值这三件事
|
||
本来就要配,也能配对。先拿样例响应把链路配通并让对方验收字段口径,等真接口就绪
|
||
只把通道改成 http 填上地址,表单上的引用一个字都不用动。
|
||
"""
|
||
started = time.perf_counter()
|
||
payload = render(db, row.mock_payload or {}, args, work=work, actor=actor)
|
||
text = json.dumps(payload, ensure_ascii=False)
|
||
result = parse(row, payload, pick=pick, extras=extras)
|
||
result.status, result.ms, result.raw = 200, _ms(started), text if raw else ""
|
||
# 样例通道根本没打外部,它失败是解析规则的事,跟上游健康无关
|
||
_mark_result(db, row, result.ok, health=False)
|
||
_log(db, row, args=args, status=result.status_label(), ms=result.ms, http_status=200,
|
||
message=result.error, actor=actor, work=work, activity=activity)
|
||
return result
|
||
|
||
|
||
def _run_process(
|
||
db: Session, row: models.DataSource, args: dict, *,
|
||
raw: bool = False, actor=None, work=None, activity=None,
|
||
pick: str = "", extras: dict | None = None
|
||
) -> FetchResult:
|
||
"""流程数据通道:查本平台另一个流程的单据,整形成一份"响应"后走**同一套解析**。
|
||
|
||
入参写法与 http 完全同源(``render`` 那一份 ``{form.x}`` / ``{auto:today}``),
|
||
所以字段上配的入参、界面上声明的入参清单,两条通道是一个说法。
|
||
|
||
刻意**不走进程内缓存**:这一条查的是本地库,一次 query 的成本远低于
|
||
"上一张单已经变了、页面还在显示五秒前的值"的代价;``cache_seconds`` 对这条通道不生效。
|
||
"""
|
||
started = time.perf_counter()
|
||
match = processdata.clean_match(render(db, row.src_match or {}, args or {}, work=work, actor=actor))
|
||
many = row.kind == models.DataSource.KIND_SEARCH
|
||
found, error = processdata.find(db, row, match, actor=actor, work=work, many=many)
|
||
ms = _ms(started)
|
||
if not found:
|
||
logger.info("data-source %s 没查到流程单据:%s", row.code, error)
|
||
_log(db, row, args=args, status=models.DataSourceLog.STATUS_ERROR, ms=ms, message=error,
|
||
actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=error, ms=ms)
|
||
payload = processdata.search_payload(found) if many else processdata.payload_of(found)
|
||
text = json.dumps(payload, ensure_ascii=False)
|
||
result = parse(row, payload, pick=pick, extras=extras)
|
||
result.status, result.ms, result.raw = 200, ms, text if raw else ""
|
||
_log(db, row, args=args, status=result.status_label(), ms=ms, http_status=0,
|
||
message=result.error, actor=actor, work=work, activity=activity)
|
||
return result
|
||
|
||
|
||
def _run_sql(
|
||
db: Session, row: models.DataSource, key: str, args: dict, *,
|
||
raw: bool = False, actor=None, work=None, activity=None,
|
||
pick: str = "", extras: dict | None = None
|
||
) -> FetchResult:
|
||
"""SQL 通道:跑配置好的那条只读查询,把结果整形成一份"响应"后走**同一套解析**。
|
||
|
||
响应形状刻意贴着 http 通道(见 ``service/sqlsource.py`` 模块头):``fetch`` 的响应
|
||
就是查出来的那一行,``search`` 的响应是 ``{"rows": […]}``。所以"取值路径"填列名、
|
||
"列表节点"填 ``rows``、"可带出的键"填别的列名——配一条 SQL 与配一个外部接口,
|
||
在表单上、在留痕里、在额度阈值里都是同一件事,没有第二套口径。
|
||
|
||
入参用 ``body_template`` 那一份(``{form.x}`` / ``{auto:today}`` 照旧),
|
||
算出来之后**按名字绑定**进 SQL,值不进语句文本。缓存与限流照吃:
|
||
前者让"审批页防抖没做好"打不到库,后者防一句重查询被搜索框点爆;
|
||
熔断不惩罚这里的失败——那多半是配置写歪了,库好得很(与 http 的 4xx 同一分寸)。
|
||
"""
|
||
started = time.perf_counter()
|
||
declared = list((row.body_template or {}).keys())
|
||
# 总开关与静态闸挡住的两种情况都记 blocked:一次库都没查,也不该算进这张源的健康度
|
||
reason = sqlsource.gate_problem()
|
||
if not reason:
|
||
problems = sqlsource.problems(row.sql_text, declared, at=f"数据源「{row.name}」的 SQL")
|
||
reason = problems[0] if problems else ""
|
||
if reason:
|
||
_log(db, row, args=args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0, message=reason,
|
||
actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=reason, health=False)
|
||
|
||
rendered = render(db, row.body_template or {}, args, work=work, actor=actor)
|
||
params = {str(name): _bind_value(value) for name, value in rendered.items()} if isinstance(rendered, dict) else {}
|
||
# 数组/对象绑不进 ``=``:选人字段给的就是人员引用数组,这句话必须在查库之前说。
|
||
# 等到"没查到数据"那一句出来,配置的人只会去怀疑自己的 WHERE,而病根在入参那一栏
|
||
container = next((str(name) for name, value in (rendered or {}).items()
|
||
if isinstance(value, (list, dict, tuple, set))), "")
|
||
if container:
|
||
reason = (f"入参 :{container} 拿到的是一个列表/对象(选人字段与多选那几格交出来的"
|
||
"就是数组),而这里所有入参都只绑标量,写成 IN 也一样会被挡。"
|
||
"要一个登录账号就把这一栏的取值写法改成 {account:form.那一格},"
|
||
"平台按人换成账号(单选取头一个);真要把一批人当条件,"
|
||
"就别拿数组绑参数——传成文本再配 FIND_IN_SET / LIKE 这类写法,"
|
||
"或者回到接口那条通道")
|
||
_log(db, row, args=args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0, message=reason,
|
||
actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=reason, health=False)
|
||
if sqlsource.RUNTIME_PARAM not in params:
|
||
# 搜索框那个关键词是运行时给的,不在配置的入参清单里;SQL 里写 LIKE CONCAT('%', :keyword, '%')
|
||
params[sqlsource.RUNTIME_PARAM] = str(args.get(sqlsource.RUNTIME_PARAM) or "")
|
||
absent = sorted(sqlsource.params(row.sql_text) - set(params))
|
||
if absent:
|
||
names = "、".join(f":{name}" for name in absent)
|
||
reason = f"这条 SQL 用到的入参 {names} 没有在入参清单里,取不到值,本次没有查库"
|
||
_log(db, row, args=args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0, message=reason,
|
||
actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=reason, health=False)
|
||
|
||
# SQL 真用了的入参这次是空的:这句话比"没查到数据"准,也省掉一次注定空手的查询
|
||
# (与 fetch.run 开头那道 missing_args 同一分寸;候选搜索的关键词空着是正常用法,不挡)
|
||
used = sqlsource.params(row.sql_text) & set(params)
|
||
blank = sorted(name for name in used
|
||
if name != sqlsource.RUNTIME_PARAM and not str(params[name] or "").strip())
|
||
if blank and row.kind != models.DataSource.KIND_SEARCH:
|
||
names = "、".join(f":{name}" for name in blank)
|
||
# 分成两种空:那一格本来就没填(等一等就好),和那一格有值但翻译不成账号
|
||
# (写法、数据对不上,等多久都一样)。混成一句会让人白等
|
||
lost = [name for name in blank if str((args or {}).get(name) or "").strip()]
|
||
if lost:
|
||
got = "、".join(f":{name}←{str((args or {}).get(name))[:60]}" for name in lost)
|
||
reason = (f"入参 {names} 翻译成了空:那一格不是没填,是这个值换不出要的东西({got})。"
|
||
"常见原因是身份表里没有这个身份、或它对应的人已经被删掉了——"
|
||
"{account:...} 只认身份 DN 与登录账号这两种值,别的它一律回空串")
|
||
else:
|
||
reason = (f"入参 {names} 这次是空的(它对应的那一格还没填),本次没有查库;"
|
||
"等那一格填上再取,这一格自然会跟着刷新")
|
||
_log(db, row, args=args, status=models.DataSourceLog.STATUS_BLOCKED, ms=0, message=reason,
|
||
actor=actor, work=work, activity=activity)
|
||
return FetchResult(ok=False, error=reason, health=False)
|
||
|
||
found = sqlsource.run(
|
||
row.sql_text, params,
|
||
many=row.kind == models.DataSource.KIND_SEARCH,
|
||
timeout_seconds=int(row.timeout_seconds or 0) or None,
|
||
)
|
||
ms = found.ms or _ms(started)
|
||
if not found.ok:
|
||
logger.info("data-source %s 的 SQL 没取到值:%s", row.code, found.error)
|
||
_finish_fail(db, row, args, started, found.error, actor, work, activity, health=False)
|
||
return FetchResult(ok=False, error=found.error, ms=ms, health=False)
|
||
|
||
payload = found.payload
|
||
text = json.dumps(payload, ensure_ascii=False)
|
||
# 「列表节点」本来是给"上游响应形状不固定"准备的;SQL 通道这份响应是我们自己拼的
|
||
# (``{"rows": […]}``),没填就按它走,别让人为一个填不出的格子去猜列名
|
||
parsed_from: Any = payload
|
||
if row.kind == models.DataSource.KIND_SEARCH and not str(row.items_path or "").strip() \
|
||
and isinstance(payload, dict):
|
||
parsed_from = payload.get("rows", payload)
|
||
result = parse(row, parsed_from, pick=pick, extras=extras)
|
||
result.status, result.ms, result.raw = 0, ms, text if raw else ""
|
||
result.note = found.note
|
||
_log(db, row, args=args, status=result.status_label(), ms=ms,
|
||
message=result.error or result.note, actor=actor, work=work, activity=activity)
|
||
if result.ok:
|
||
_cache_put(key, int(row.cache_seconds or 0), result)
|
||
return result
|
||
|
||
|
||
def _bind_value(value: Any) -> Any:
|
||
"""送进数据库绑定的那一格:只给标量。
|
||
|
||
选了多选、选了部门树那种把数组/对象当入参传给 ``=`` 是配错,不是"另一种查法",
|
||
与其让 pymysql 抛一句 ``unsupported format`` ,不如在这里洗成文本:
|
||
SQL 里想按数组查,该写 ``FIND_IN_SET(:tags, ...)`` 或 ``IN`` 的自己拼不出安全写法,
|
||
那种需求本来就该回到接口那条通道。
|
||
"""
|
||
if value is None or isinstance(value, (str, int, float, bool)):
|
||
return value
|
||
return json.dumps(value, ensure_ascii=False)
|
||
|
||
|
||
def _finish_fail(
|
||
db: Session, row: models.DataSource, args: dict, started: float, message: str,
|
||
actor, work, activity, http_status: int = 0, health: bool = True
|
||
) -> None:
|
||
_mark_result(db, row, False, health)
|
||
logger.warning("data-source %s 调用失败:%s", row.code, message)
|
||
_log(db, row, args=args, status=models.DataSourceLog.STATUS_ERROR, ms=_ms(started),
|
||
http_status=http_status, message=message, actor=actor, work=work, activity=activity)
|
||
|
||
|
||
def parse(
|
||
row: models.DataSource, payload: Any, *,
|
||
pick: str | None = None, extras: dict | None = None
|
||
) -> FetchResult:
|
||
"""按数据源声明的解析规则把响应变成「值 + 可带出的键」或「一份列表」。
|
||
|
||
``pick`` / ``extras`` 是**字段上那份覆盖**(``fetch.pick`` / ``fetch.extras``):
|
||
一张源常常回好几列,而"这一格要哪一列"是这张表单的事,不该逼着人回「外部数据源」页
|
||
给每个用法另建一条源,或者去改一张已被别的字段用着的源(改了它们一起变)。
|
||
留空 = 不覆盖,照源上那一份走;``extras`` 是**并**上源上声明的那些,同名以字段上的为准。
|
||
"""
|
||
if row.kind == models.DataSource.KIND_SEARCH:
|
||
rows = pick_path(payload, row.items_path) if row.items_path else payload
|
||
if not isinstance(rows, list):
|
||
where = (pick_miss(payload, row.items_path) or f"响应里 {row.items_path} 不是列表") \
|
||
if row.items_path else "响应本身不是列表,列表节点(如 data.rows)得填上"
|
||
# 上游是答话了的,只是这一格的解析规则或入参对不上它的形状:这是我们这侧的事
|
||
return FetchResult(ok=False, error=where, health=False)
|
||
items: list[dict] = []
|
||
for item in rows:
|
||
if not isinstance(item, dict):
|
||
items.append({"value": str(item), "label": str(item), "extra": {}})
|
||
continue
|
||
extra = {name: pick_path(item, path) for name, path in _extra_paths(row, extras).items()}
|
||
value = pick_path(item, row.map_value or "value")
|
||
label = pick_path(item, row.map_label or "label")
|
||
items.append({"value": "" if value is None else str(value),
|
||
"label": str(label if label not in (None, "") else value),
|
||
"extra": extra})
|
||
return FetchResult(ok=True, items=items)
|
||
|
||
path = str(pick or "").strip() or str(row.pick or "")
|
||
value = pick_path(payload, path) if path else payload
|
||
if value is None:
|
||
where = (pick_miss(payload, path) or f"响应里没有 {path} 这个值") if path \
|
||
else "响应是空的,取值路径得填上(如 data.balance)"
|
||
# 上游答了话、却说这事没成(项目不存在、编码为空……):那句人话比"取不到值"有用得多。
|
||
# 但只在**业务位真的不对**时才让它盖过断点提示——配了 success_path 的源十有八九
|
||
# 会把 message 写成 "success",业务明明成功还拿它盖掉"你筛的那个值不存在"那句话,
|
||
# 界面就只剩一句让人摸不着头脑的"上游说:success"(判据与出站同一份:outbox._verdict)
|
||
hint = pick_path(payload, row.message_path) if row.message_path else None
|
||
if isinstance(hint, str) and hint.strip() and _business_rejected(row, payload):
|
||
where = f"上游说:{hint.strip()}"
|
||
# 同上:答话了,只是没给出要的那个值——多半是入参或解析规则的事,不算上游病了
|
||
return FetchResult(ok=False, error=where, health=False)
|
||
merged = {name: pick_path(payload, path) for name, path in _extra_paths(row, extras).items()}
|
||
return FetchResult(ok=True, value=value, extras=merged)
|
||
|
||
|
||
def _business_rejected(row: models.DataSource, payload: Any) -> bool:
|
||
"""这一份响应是不是"业务上被拒了":配了成功判定、而成功位不等于期望值。
|
||
|
||
没配成功判定的源回 ``False``——那时我们无从判断,那句 ``message`` 也就可以是任意一句话,
|
||
拿它盖掉精确的断点提示是赌。比较口径照 :func:`app.service.outbox._verdict` 那一份来,
|
||
两边各判一次迟早会漂。
|
||
"""
|
||
if not str(row.success_path or "").strip():
|
||
return False
|
||
got = pick_path(payload, row.success_path)
|
||
return str(got if got is not None else "").strip() != str(row.success_value or "").strip()
|
||
|
||
|
||
def _extra_paths(row: models.DataSource, extra: dict | None) -> dict:
|
||
"""这一趟要解析哪些可带出键:源上声明的那几张表,并上字段上自己补的那几行。"""
|
||
out = {str(name): str(path) for name, path in (row.map_extra or {}).items()}
|
||
for name, path in (extra or {}).items():
|
||
key, target = str(name or "").strip(), str(path or "").strip()
|
||
if key and target:
|
||
out[key] = target
|
||
return out
|
||
|
||
|
||
def _body_has_keyword(row: models.DataSource) -> bool:
|
||
return isinstance(row.body_template, dict) and "keyword" in json.dumps(row.body_template, ensure_ascii=False)
|
||
|
||
|
||
def _may_call(db: Session, actor: Any, permission: str) -> bool:
|
||
"""这个人能不能调这个源。
|
||
|
||
接口链路上拿到的是 ``CurrentUser``,权限码已经解好了;引擎链路(提交时服务端复核
|
||
那一次就走这里)拿到的是 ``Actor``——那只是一个「人 + 身份」,**没有权限码**。
|
||
过去这等于「凡是配了调用权限的源,在引擎链路里一律调不到」,于是页面上明明显示过
|
||
的值会在提交这一刻被判成失败,而且两处口径不一致比两处都失败更难查。
|
||
所以没有权限码的那种 actor 就地按人解一次;只有真配了调用权限的源才会走到这句。
|
||
"""
|
||
if getattr(actor, "is_admin", False):
|
||
return True
|
||
codes = getattr(actor, "permissions", None)
|
||
if codes is None:
|
||
person_id = str(getattr(actor, "person_id", "") or "").strip()
|
||
if not person_id:
|
||
return False
|
||
codes, _, _ = perm.resolve_grant(db, person_id)
|
||
return permission in set(codes or ())
|
||
|
||
|
||
def _ms(started: float) -> int:
|
||
return int((time.perf_counter() - started) * 1000)
|
||
|
||
|
||
def _brief(text: str, limit: int = 200) -> str:
|
||
flat = " ".join(str(text or "").split())
|
||
return flat if len(flat) <= limit else flat[:limit] + "…"
|
||
|
||
|
||
def friendly_error(exc: Exception) -> str:
|
||
reason = getattr(exc, "reason", None) or exc
|
||
text = str(reason)
|
||
if isinstance(exc, TimeoutError) or "timed out" in text.lower():
|
||
return "外部系统响应超时"
|
||
return f"外部系统连不上:{text}"
|
||
|
||
|
||
def used_by_forms(db: Session, code: str) -> list[str]:
|
||
"""哪些表单正在引用这个数据源。删之前先提醒一句,别把填单页面点成空白。
|
||
|
||
两条引用路都算:字段上的外部取数(``fetch.sources[].source``),以及
|
||
候选来源=外部数据源(``dataSourceCode``)——后者删了同样是那一格再也搜不出东西。
|
||
"""
|
||
if not code:
|
||
return []
|
||
want = str(code).strip()
|
||
hits: list[str] = []
|
||
rows = db.execute(
|
||
select(models.FormElement.id, models.FormElement.name, models.FormElement.content)
|
||
).all()
|
||
for form_id, name, content in rows:
|
||
for field_def in _walk_fields(content):
|
||
# 候选来源=外部数据源那两格都算:字段上那一份(兜底),以及条件路由里每一条挂的源
|
||
codes = {str(field_def.get("dataSourceCode") or "").strip()}
|
||
routes = field_def.get("optionRoutes")
|
||
if isinstance(routes, list):
|
||
codes.update(
|
||
str(item.get("dataSourceCode") or "").strip()
|
||
for item in routes
|
||
if isinstance(item, dict)
|
||
)
|
||
referenced = any(
|
||
entry["source"] == want for entry in binding.entries(field_def.get(FIELD_PROP))
|
||
) or want in codes
|
||
if referenced:
|
||
hits.append(name or form_id)
|
||
break
|
||
return hits
|
||
|
||
|
||
def _walk_fields(content: object) -> list[dict]:
|
||
"""表单内容里的字段定义,兼容 {fields: [...]} 与裸数组两种存法。"""
|
||
if isinstance(content, dict):
|
||
return [item for item in (content.get("fields") or []) if isinstance(item, dict)]
|
||
if isinstance(content, list):
|
||
return [item for item in content if isinstance(item, dict)]
|
||
return []
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 字段配置:读取、提交时覆盖、设计期校验
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def fetch_fields(fields: Iterable[dict] | None, data: dict | None = None) -> list[tuple[dict, dict]]:
|
||
"""挑出配了 ``fetch`` 的**主表字段**,返回 (字段定义, 取数配置)。
|
||
|
||
给了 ``data`` 就按**这一单此刻**那一份配置挑(候选路由自带的带出也算配了取数);
|
||
不给就只看字段上那一份 ``fetch``——设计期那份清单本来就不依赖取值。
|
||
|
||
明细表那一格不在这份清单里:列上那份 ``fetch`` 是逐行跑的,问的是
|
||
:func:`row_fetch_fields`。这一句只回主表,老调用方(试跑、留痕、权限判断)
|
||
不用先自己滤掉 ``table``。
|
||
"""
|
||
out: list[tuple[dict, dict]] = []
|
||
for item in fields or []:
|
||
if str(item.get("type") or "") == TABLE_TYPE:
|
||
continue
|
||
cfg = config_of(item, data)
|
||
if isinstance(cfg, dict) and field_sources(cfg):
|
||
out.append((item, cfg))
|
||
return out
|
||
|
||
|
||
def table_columns(table: dict) -> list[dict]:
|
||
"""明细表配了哪几列(与 ``formruntime.columns_of`` 同一份读法,这里不模块级 import 服务层)。"""
|
||
raw = table.get("columns")
|
||
return [item for item in raw if isinstance(item, dict)] if isinstance(raw, list) else []
|
||
|
||
|
||
def row_fetch_fields(fields: Iterable[dict] | None, data: dict | None = None) -> list[tuple[dict, dict, dict]]:
|
||
"""明细表里配了 ``fetch`` 的那些列,返回 (表定义, 列定义, 取数配置)。
|
||
|
||
列上那份 ``fetch`` 与主表字段那一份是**同一个形状、同一个执行器**(``binding.entries``
|
||
那一套多源 / 入参 / 带出 / 失败策略),只差两件事,这两件事正是明细行与主表那一格的
|
||
分界:
|
||
|
||
* **入参读这一行**:作用域是「整张单 + 这一行」,与主表字段同名的列以本行为准——
|
||
跟「本行公式」吃的是同一份作用域(``formRuntime.applyRowFormulas``),配的人不用
|
||
再记第二套写法;``{form.x}`` 引用的既可以是本行的列,也可以是主表字段。
|
||
* **带出只落在这张表**:一行生一格,把 ``picks`` 写进主表等于让最后一行说了算,
|
||
所以带出去向只认这张明细表的其它列。
|
||
"""
|
||
out: list[tuple[dict, dict, dict]] = []
|
||
for item in fields or []:
|
||
if str(item.get("type") or "") != TABLE_TYPE:
|
||
continue
|
||
for column in table_columns(item):
|
||
cfg = config_of(column, data)
|
||
if isinstance(cfg, dict) and field_sources(cfg):
|
||
out.append((item, column, cfg))
|
||
return out
|
||
|
||
|
||
def field_sources(cfg: dict) -> list[dict]:
|
||
"""这一格要调的数据源清单;形状与归一规则在 ``service/binding.entries``。"""
|
||
return binding.entries(cfg)
|
||
|
||
|
||
#: 这一份取数**只写带出去向**、不动这一格自己那一个值(由 :func:`fill_config` 合成时打上,
|
||
#: 不落到存下来的表单定义里)。
|
||
FILL_ONLY_KEY = "fillOnly"
|
||
|
||
|
||
def _linkage():
|
||
"""候选来源那一层(延迟导入:``dicts`` 反过来也读这一层的取值写法)。"""
|
||
from . import dicts as _dicts
|
||
|
||
return _dicts
|
||
|
||
|
||
def fill_config(field: dict, data: dict | None) -> dict | None:
|
||
"""把**此刻命中的那一条候选路由**自带的带出(``route.fill``)摊成一份取数配置。
|
||
|
||
源与条件都从那一条来(``dataSourceCode`` / ``when``),入参没另写时沿用那一条筛候选用的
|
||
入参;打上 :data:`FILL_ONLY_KEY`,于是它只往「带出去向」那几格写。失败策略、发起后冻结、
|
||
多久算旧这三档沿用字段上那一份 ``fetch``——它们问的是「这一格怎么待人」,跟从哪家取数无关。
|
||
|
||
没命中、命中的那条没自带带出,一律回 ``None``(= 这一格还是按字段上那一份 ``fetch`` 跑,
|
||
老表单一个字不变)。对面是前端 ``utils/formFetch.fetchConfigOf``。
|
||
"""
|
||
linkage = _linkage()
|
||
route = linkage.matched_route(field, data)
|
||
entry = linkage.route_fill_entry(route) if route else None
|
||
if not entry:
|
||
return None
|
||
cfg: dict = {**entry, FILL_ONLY_KEY: True}
|
||
own = field.get(FIELD_PROP)
|
||
if isinstance(own, dict):
|
||
for key in ("onError", "freezeAfterStart", "revalidateAfter", "separator"):
|
||
if own.get(key) not in (None, ""):
|
||
cfg[key] = own[key]
|
||
return cfg if field_sources(cfg) else None
|
||
|
||
|
||
def config_of(field: dict, data: dict | None = None) -> dict | None:
|
||
"""这一格**此刻**那份取数配置:命中那条自带带出就用它摊出来的那份,否则用字段上那一份。
|
||
|
||
与候选来源那份回落同一个分寸:一条都不命中、或命中的那条没自带带出,读的就是
|
||
``field["fetch"]``(改动前行为)。
|
||
"""
|
||
return fill_config(field, data) or field.get(FIELD_PROP)
|
||
|
||
|
||
def has_fetch(field: dict) -> bool:
|
||
"""这一格到底会不会往外写值:字段自己配了 ``fetch``,或某一条候选路由自带了带出。
|
||
|
||
清单与依赖图这类「有没有配」的判断要用这一句(不看此刻的取值);真正跑哪一份问
|
||
:func:`config_of`。
|
||
"""
|
||
cfg = field.get(FIELD_PROP)
|
||
if isinstance(cfg, dict) and field_sources(cfg):
|
||
return True
|
||
linkage = _linkage()
|
||
return any(linkage.route_fill_entry(route) for route in linkage.routes_of(field))
|
||
|
||
|
||
def active_field_sources(cfg: dict, scope: dict | None) -> list[dict]:
|
||
"""这一格**此刻**要调的那几张源:条目上配了触发条件的,只留成立的那几条。
|
||
|
||
「费用类型 = 差旅才问差标接口」就靠这一句生效。设计期检查(``problems``)看的是全量清单
|
||
——源在不在、入参引用掉没掉在地上,跟此刻的取值无关;运行时才按 ``scope`` 筛。
|
||
判据与联动条件同一份(``service/dicts.condition_ok``),不会界面上调了、提交时没调。
|
||
"""
|
||
return binding.active_entries(cfg, scope)
|
||
|
||
|
||
def applies_at(cfg: dict | None, activity=None) -> bool:
|
||
"""这一格现在还可以取数吗(``fetch.freezeAfterStart`` 那一档)。
|
||
|
||
默认每趟提交都取(行为与配了取数以来的每一张单一个字不差)。勾了这一档的那一格
|
||
在**流程真正发起之前**照旧随时取:发起页、草稿、停在开始环节等提交的那一单、
|
||
退回到发起人重填的那一单,全都还在"没发起"这一侧,入参一变照样带值。
|
||
单子一旦离开开始环节进了办理环节,就不再调这个接口,这一格与它带出去的那几格
|
||
都留在发起那一刻取回的值上——外部系统那边的余额、汇率、主数据随时在动,让某
|
||
一环的提交顺手改掉经办人当初填熟的那几格,审批人就看不见"申请人提交时到底带回
|
||
来什么",``form_change_notes`` 还会把这次覆盖记成办理人改的数据。
|
||
|
||
没有环节上下文(接口直接建单、发起页上那次预取)按"还没发起"算——这一档要保的
|
||
是「在路上别被外部值改数据」,不是「一次都不许多取」。
|
||
"""
|
||
if not bool((cfg or {}).get(FREEZE_AFTER_START)):
|
||
return True
|
||
return str(getattr(activity, "type", "") or "") in ("", BEGIN_TYPE)
|
||
|
||
|
||
def _entry_extras(entry: dict) -> dict | None:
|
||
"""这一格自己补的那几行带出列(``fetch.extras``),没补就回 ``None``。"""
|
||
rows = entry.get("extras")
|
||
return rows if isinstance(rows, dict) else None
|
||
|
||
|
||
def merge_values(mode: str, values: list[Any], *, separator: str = "、") -> Any:
|
||
"""把几个接口回来的值合成一格的值;口径在 ``service/binding.merge``。"""
|
||
return binding.merge(mode, values, separator=separator)
|
||
|
||
|
||
def resolve_args(db: Session, cfg: dict, data: dict, work=None, actor=None) -> dict:
|
||
"""把字段上配的入参写法算成这次调用的入参。"""
|
||
return binding.resolve_args(db, cfg, data, work=work, actor=actor)
|
||
|
||
|
||
def coerce(field_type: str, value: Any) -> Any:
|
||
"""按目标字段的类型塑形:金额格要拿到数字,不是 \"12,300.00\"。"""
|
||
if value is None:
|
||
return None
|
||
if field_type in MONEY_TYPES:
|
||
cents = binding.to_cents(value)
|
||
return binding.from_cents(cents) if cents is not None else value
|
||
if isinstance(value, (dict, list)):
|
||
return json.dumps(value, ensure_ascii=False)
|
||
return value
|
||
|
||
|
||
def apply_form(
|
||
db: Session,
|
||
fields: Iterable[dict] | None,
|
||
data: dict,
|
||
*,
|
||
work=None,
|
||
actor=None,
|
||
activity=None,
|
||
perms: dict | None = None,
|
||
) -> None:
|
||
"""提交链路里的外部取数:**以取回的值为准覆盖前端**。
|
||
|
||
与 ``dicts.apply_auto_fill`` 同一分寸——外部回来的字段如果能被前端随便塞,
|
||
「余额充足」这四个字就没有存在的意义。位置在公式之前,
|
||
这样「余额 - 申请金额 >= 0」这类公式与后面的条件分支读的是同一份数。
|
||
|
||
``perms`` 是这一环展开后的字段权限(``formruntime.perm_map``,调用方现成有一份)。
|
||
它只决定一件事:这一格在这一环**露不露**——露着才谈得上"取不到就别让他提交"。
|
||
藏起来的格子那一屏既看不见它也改不动它,``onError=block`` 在这一环一律降级成不挡、
|
||
值留在原来那一份上(与阈值/额度同一口径,见 ``threshold.blocked_here``);
|
||
**取数照旧发起**,条件分支与下一环节读的还是新数。没给 ``perms`` 时按露着算。
|
||
"""
|
||
rows = [item for item in (fields or []) if isinstance(item, dict)]
|
||
entries = fetch_fields(rows, data)
|
||
# 明细行那一趟也算在内:只给明细列配了取数的表单,主表这一趟是空的,不能提前返回
|
||
row_targets = row_fetch_fields(rows, data)
|
||
if not entries and not row_targets:
|
||
return
|
||
|
||
# 带出去向只能写进这一份字段清单里真存在的格子:格子删了而带出还留着的老配置,
|
||
# 键名(field_text_2 这种)日后很容易被新字段复用,悄悄盖上去比少写一格危险。
|
||
keys = binding.field_keys(rows)
|
||
trace = dict(data.get(TRACE_KEY) or {})
|
||
for field_def, cfg in entries:
|
||
key = str(field_def.get("key") or "")
|
||
label = str(field_def.get("label") or key)
|
||
# 「发起之后冻结」:进了办理环节就不再调这个接口,这一格与它带出去的那几格
|
||
# 都留在发起那一刻取回来的值上(连留痕一起留住,不抹掉来路)
|
||
if not applies_at(cfg, activity):
|
||
continue
|
||
_apply_cell(
|
||
db, cfg, field_def, data, keys, data, trace,
|
||
at=f"「{label}」", trace_key=key, policy=_policy_of(cfg, field_def, perms, data),
|
||
work=work, actor=actor, activity=activity,
|
||
)
|
||
# 明细表里那些配了取数的列:同一份配置,逐行各跑一次,值写进这一行
|
||
_apply_rows(db, row_targets, data, trace, work=work, actor=actor, activity=activity, perms=perms)
|
||
if trace:
|
||
data[TRACE_KEY] = trace
|
||
|
||
|
||
def _policy_of(cfg: dict, field_def: dict, perms: dict | None, data: dict) -> str:
|
||
"""这一格取不到数时怎么兜(``keep`` / ``block`` / ``zero``),本环节藏着的格子先降级。
|
||
|
||
主表那一格与明细行那一格共用这一句:被本环节藏起来的格子那一屏既看不见它也改不动它,
|
||
``onError=block`` 在这一律降级成不挡、值留在原来那一份上(与阈值/额度同一口径,
|
||
见 ``threshold.blocked_here``);**取数照旧发起**,条件分支与下一环节读的还是新数。
|
||
明细行里那一列露不露脸,看的是它所属那张明细表在这一环露不露(环节权限挂的是表)。
|
||
"""
|
||
from . import formruntime # 局部导入:与 apply_form 同一处理
|
||
|
||
policy = str(cfg.get("onError") or "keep").strip()
|
||
if policy != "block" or not formruntime.field_hidden(field_def, perms, data):
|
||
return policy
|
||
label = str(field_def.get("label") or field_def.get("key") or "")
|
||
logger.info("外部取数失败但这一环不挡提交:%s 这一格在当前环节没显示", label)
|
||
return "keep"
|
||
|
||
|
||
def _run_entries(
|
||
db: Session, cfg: dict, scope: dict, keys: set[str], target: dict, *,
|
||
work=None, actor=None, activity=None, at: str = "", own_key: str = ""
|
||
) -> tuple[list[Any], list[str], list[dict]]:
|
||
"""跑这一格配的那几张源:值收进一份清单,每张源配的带出去向顺手写进 ``target``。
|
||
|
||
主表那一格传的 ``scope`` 与 ``target`` 都是 ``data``(老行为一个字不动:后一个源的入参
|
||
读得到前一个源刚带进 ``data`` 的那几格);明细行那一格传的是「整张单 + 这一行」这一份
|
||
作用域,以及那一行本身。两条路共用同一次外呼、同一份留痕形状、同一条「带出去向那一格
|
||
还在不在」的闸(``binding.live_target``)——差的只是往哪个对象里写。
|
||
|
||
``own_key`` 给成这一格自己的字段名时,带出去向里指着它的那几行**不写**:候选路由自带的
|
||
那一份带出只准往别的格子上写,这一格那一个值是人从候选里挑的(见 :data:`FILL_ONLY_KEY`)。
|
||
设计期已经把这种配法报成毛病,这里是兜住老配置与接口直传那两条路。
|
||
|
||
回 ``(values, failures, called)``:怎么合成一个值、取不到怎么兜,归调用方(``_apply_cell``)。
|
||
"""
|
||
values: list[Any] = []
|
||
failures: list[str] = []
|
||
called: list[dict] = []
|
||
# 只跑**此刻触发条件成立**的那几条(``when``,见 ``active_field_sources``):
|
||
# 一条都没触发就是这一格这回不取数,与「配了源但源停了」不是一回事,不记失败
|
||
for entry in binding.active_entries(cfg, scope):
|
||
code = str(entry.get("source") or "")
|
||
row = get_source(db, code)
|
||
if row is None or not row.enabled:
|
||
failures.append(f"{code}:未配置或已停用")
|
||
continue
|
||
args = resolve_args(db, entry, scope, work, actor)
|
||
result = run(db, row, args, actor=actor, work=work, activity=activity,
|
||
pick=str(entry.get("pick") or ""), extras=_entry_extras(entry))
|
||
if not result.ok:
|
||
failures.append(f"{row.name}:{result.error}")
|
||
continue
|
||
values.append(result.value)
|
||
for source_key, pick_target in entry["picks"].items():
|
||
target_key = binding.live_target(pick_target, keys)
|
||
if own_key and target_key == own_key:
|
||
continue
|
||
if not target_key:
|
||
if str(pick_target or "").strip():
|
||
logger.warning("字段 %s 的带出指向表单里已经没有的格子 %s,这一笔没写",
|
||
at or code, pick_target)
|
||
continue
|
||
value = result.extras.get(str(source_key)) if str(source_key) in (result.extras or {}) else None
|
||
if value is None and str(source_key) == "value":
|
||
value = result.value
|
||
target[target_key] = value
|
||
called.append({"source": code, "args": args, "cached": result.cached,
|
||
"at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
# 这一格自己指定了取哪一列时留一笔,事后看留痕才知道读的是哪条路径
|
||
**({"pick": str(entry.get("pick"))} if str(entry.get("pick") or "").strip() else {})})
|
||
return values, failures, called
|
||
|
||
|
||
def _apply_cell(
|
||
db: Session, cfg: dict, field_def: dict, scope: dict, keys: set[str], target: dict,
|
||
trace: dict, *, at: str, trace_key: str, policy: str,
|
||
work=None, actor=None, activity=None
|
||
) -> tuple[bool, list[str]]:
|
||
"""按这一格那份 ``fetch`` 把值配好:写进 ``target``,带出也写进 ``target``,留痕记进 ``trace``。
|
||
|
||
主表那一格(``target`` 是 ``data``)与明细表的一格(``target`` 是那一行)走的就是这一个
|
||
函数,多源合并、失败策略、留痕形状因此不可能走偏成两套。``at`` 是报错时的门牌,
|
||
``trace_key`` 是留痕里这一格的名字,``extra_trace`` 给明细行多带的那一格(行号)。
|
||
|
||
回 ``(这一格有没有写上值, 这一格没取到的那几句)``:值写进 ``target``,带出也写进
|
||
``target``,留痕记进 ``trace[trace_key]``。
|
||
"""
|
||
key = str(field_def.get("key") or "")
|
||
merge = str(cfg.get("merge") or "first").strip()
|
||
separator = str(cfg.get("separator") or "、")
|
||
values, failures, called = _run_entries(
|
||
db, cfg, scope, keys, target, work=work, actor=actor, activity=activity, at=at,
|
||
own_key=key if cfg.get(FILL_ONLY_KEY) else "",
|
||
)
|
||
blank_value = 0 if str(field_def.get("type") or "") in MONEY_TYPES else ""
|
||
if cfg.get(FILL_ONLY_KEY):
|
||
# 候选路由自带的这一条只管「带出去向」那几格(``_run_entries`` 已经写进 target 了):
|
||
# 这一格自己那一个值是人从候选里挑的,不许被取回来的数盖掉,也不参与合并;
|
||
# 取不到时 ``block`` 照旧挡提交,``zero`` 不动这一格(要清的本来就不是它)。
|
||
if failures and policy == "block":
|
||
raise _engine_error()(f"{at}的外部数据取不到:{';'.join(failures)}")
|
||
if called:
|
||
first = called[0]
|
||
trace[trace_key] = {"source": first["source"], "args": first["args"], "cached": first["cached"],
|
||
"at": first["at"], "merge": "", "sources": called,
|
||
**({"failed": failures} if failures else {})}
|
||
if failures:
|
||
logger.warning("字段 %s 的候选路由带出有一部分没取到:%s", key, ";".join(failures))
|
||
return False, failures
|
||
if not called:
|
||
if failures and policy == "block":
|
||
raise _engine_error()(f"{at}的外部数据取不到:{';'.join(failures)}")
|
||
if failures and policy == "zero":
|
||
target[key] = blank_value
|
||
return False, failures
|
||
# 少调成一个源,这格的值就是残缺的。只有 first 那档本来就允许"轮到谁算谁",
|
||
# 其余(相加 / 取最小…)凑不齐就当这次取数失败,按 onError 处理,
|
||
# 免得把「三个系统的余额之和」悄悄写成一个少了某一家的数。
|
||
if failures and merge != "first":
|
||
if policy == "block":
|
||
raise _engine_error()(f"{at}的外部数据只取到一部分:{';'.join(failures)}")
|
||
if policy == "zero":
|
||
target[key] = blank_value
|
||
return False, failures
|
||
merged = merge_values(merge, values, separator=separator)
|
||
if merged is None:
|
||
return False, failures
|
||
target[key] = coerce(str(field_def.get("type") or ""), merged)
|
||
first = called[0]
|
||
trace[trace_key] = {"source": first["source"], "args": first["args"], "cached": first["cached"],
|
||
"at": first["at"], "merge": merge if len(called) > 1 else "",
|
||
"sources": called, **({"failed": failures} if failures else {})}
|
||
if failures and len(called) > 1:
|
||
# 部分成功也得看得见:留痕里挂着失败的是哪几家,日志里也留一句
|
||
logger.warning("字段 %s 的多源取数有部分没成:%s", key, ";".join(failures))
|
||
return True, failures
|
||
|
||
|
||
def _apply_rows(
|
||
db: Session, targets: list[tuple[dict, dict, dict]], data: dict, trace: dict, *,
|
||
work=None, actor=None, activity=None, perms: dict | None = None
|
||
) -> None:
|
||
"""明细表里那些配了取数的列:逐行跑一次,值写进**这一行**的这一列。
|
||
|
||
``targets`` 是 :func:`row_fetch_fields` 挑出来的那几列(表定义、列定义、取数配置)。
|
||
|
||
行按 ``data`` 里当前那几行跑(``normalise_rows`` 已经整过型、丢过空行),所以被人删掉的
|
||
行不会再取一次数,新加的那一行当场补得上。兜法与主表那一格一模一样(同一个
|
||
:func:`_apply_cell`),只是报错要点名是哪一行:
|
||
「明细表「收款明细」第 2 行的「银行帐号」的外部数据取不到」。
|
||
|
||
留痕按「表名.列名」记一条,底下挂每一行那次外呼的入参与时刻——两张明细表可能有同名列,
|
||
同一张表每一行都得留得开,所以不能沿用主表那份「列名当键」。
|
||
"""
|
||
for table, column, cfg in targets:
|
||
table_key = str(table.get("key") or "")
|
||
table_label = str(table.get("label") or table_key)
|
||
column_key = str(column.get("key") or "")
|
||
column_label = str(column.get("label") or column_key)
|
||
rows_value = data.get(table_key)
|
||
if not table_key or not column_key or not applies_at(cfg, activity):
|
||
continue
|
||
if not isinstance(rows_value, list) or not rows_value:
|
||
continue
|
||
# 带出去向只认这张表的列:写进主表那一格等于让最后一行去盖表头
|
||
column_keys = {str(item.get("key") or "").strip() for item in table_columns(table)}
|
||
column_keys.discard("")
|
||
policy = _policy_of(cfg, table, perms, data)
|
||
row_trace: dict[str, Any] = {}
|
||
failed: list[str] = []
|
||
for index, row in enumerate(rows_value, start=1):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
# 入参读的是「整张单 + 这一行」,同名的列以本行为准——与本行公式同一份作用域
|
||
scope = {**data, **row}
|
||
at = f"明细表「{table_label}」第 {index} 行的「{column_label}」"
|
||
# 列上那份候选路由自带的带出,命中与否看的是**这一行**里的值
|
||
_wrote, failures = _apply_cell(
|
||
db, config_of(column, scope) or cfg, column, scope, column_keys, row, row_trace,
|
||
at=at, trace_key=str(index), policy=policy,
|
||
work=work, actor=actor, activity=activity,
|
||
)
|
||
failed.extend(f"第 {index} 行:{text}" for text in failures)
|
||
if not row_trace:
|
||
continue
|
||
calls: list[dict] = []
|
||
# 每行那份留痕按行号记,摊平成一张清单时把行号写进每一次调用里
|
||
for row_no, item in sorted(row_trace.items(), key=lambda kv: int(kv[0])):
|
||
for call in item.get("sources") or []:
|
||
calls.append({"row": int(row_no),
|
||
**{key: call.get(key) for key in ("source", "args", "cached", "at")}})
|
||
first = row_trace[min(row_trace, key=int)]
|
||
trace[f"{table_key}.{column_key}"] = {
|
||
"source": first.get("source"), "args": first.get("args"), "cached": first.get("cached"),
|
||
"at": first.get("at"), "merge": first.get("merge", ""), "rows": calls,
|
||
**({"failed": failed} if failed else {}),
|
||
}
|
||
|
||
|
||
def _engine_error():
|
||
"""延迟导入:``app.engine.engine`` 会 import 本模块,顶层导会成环。"""
|
||
from ..engine import engine as engine_module
|
||
|
||
return engine_module.EngineError
|
||
|
||
|
||
def preview(db: Session, row: models.DataSource, args: dict, actor=None) -> dict:
|
||
"""设计器与「测试连通」用:真调一次,把原始响应与解析结果一起回。
|
||
|
||
解析规则**不用另外传**:``run`` 里 ``pick`` 留空就落到 ``row.pick``,而「拉取样例」
|
||
送进来的 ``row`` 本来就是弹窗里那份没保存的草稿(``api/data_sources.py:_draft``),
|
||
所以刚敲进去的那条取值路径当场就会被验一次——这一句是这里的行为约定,别再改回去。
|
||
"""
|
||
result = run(db, row, args, actor=actor, raw=True)
|
||
return {
|
||
"ok": result.ok,
|
||
"value": result.value,
|
||
"extras": result.extras,
|
||
"items": result.items[:200],
|
||
"itemCount": len(result.items),
|
||
"error": result.error,
|
||
"note": result.note,
|
||
"status": result.status,
|
||
"ms": result.ms,
|
||
"cached": result.cached,
|
||
"raw": _brief(result.raw, 4000),
|
||
"paths": sample_paths(_safe_json(result.raw)),
|
||
# 拿到真实响应的这一刻顺手反推一份解析配置:界面上一键贴上去,
|
||
# 省掉「看着 JSON 手打 data.rows.0.balance」这一段最容易配错的活
|
||
"suggestions": presets.suggest(_safe_json(result.raw), row.kind),
|
||
}
|
||
|
||
|
||
def _safe_json(text: str) -> Any:
|
||
try:
|
||
return json.loads(text or "")
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def sample_paths(payload: Any, prefix: str = "", depth: int = 5) -> list[dict]:
|
||
"""把样例响应摊成一条条可点选的取值路径,界面上一边看样例一边点。
|
||
|
||
只列标量叶子与列表节点:让管理员在界面上手打 ``data.rows.0.balance``
|
||
这种字符串路径,配错只是少了一格,配对了也没人敢信。
|
||
|
||
深度是 5 不是 4:``{code, message, data: {rows: [{amount: …}]}}`` 这一派带外壳的
|
||
列表响应,要走到行里那一列正好要五层(根 → data → rows → 第 0 行 → amount),
|
||
四层时列表点得开、行里那一列却一条都不出现,而"从明细列表里挑一个数"恰恰是
|
||
最常见的那个用法。底下 ``[:200]`` 才是真上限,不怕它摊开。
|
||
|
||
空响应(不是 JSON、或响应体本来就是空的)回空列表:``None`` 往下走会产出一条
|
||
``path`` 为空串的幽灵行,界面上就是一个字都看不见的行配两颗按钮,
|
||
比一句"还没有样例响应"更像程序坏了。
|
||
"""
|
||
out: list[dict] = []
|
||
if depth <= 0:
|
||
return out
|
||
if isinstance(payload, dict):
|
||
for key, value in payload.items():
|
||
path = f"{prefix}.{key}" if prefix else str(key)
|
||
out.extend(sample_paths(value, path, depth - 1))
|
||
elif isinstance(payload, list):
|
||
out.append({"path": prefix, "kind": "list", "count": len(payload),
|
||
"sample": f"[{len(payload)} 项]"})
|
||
if payload:
|
||
out.extend(sample_paths(payload[0], f"{prefix}.0", depth - 1))
|
||
elif prefix:
|
||
out.append({"path": prefix, "kind": type_name(payload), "sample": _brief(str(payload if payload is not None else ""), 80)})
|
||
return out[:200]
|
||
|
||
|
||
def type_name(value: Any) -> str:
|
||
if isinstance(value, bool):
|
||
return "boolean"
|
||
if isinstance(value, (int, float)):
|
||
return "number"
|
||
if value is None:
|
||
return "null"
|
||
return "string"
|
||
|
||
|
||
def problems(fields: Iterable[dict] | None, db: Session | None = None) -> list[str]:
|
||
"""字段上 ``fetch`` 配置的毛病。与 ``quota.problems`` 同一把尺子,前端有镜像。
|
||
|
||
明细表的列也在这儿判:列上那份 ``fetch`` 与主表字段那份跑的是同一个执行器,
|
||
判据自然共用同一份(见 :func:`_cfg_problems`),只是门牌要说清是哪张表的哪一列,
|
||
而带出去向那一栏只认这张表的列。
|
||
"""
|
||
out: list[str] = []
|
||
rows = [item for item in (fields or []) if isinstance(item, dict)]
|
||
keys = {str(item.get("key") or "").strip() for item in rows}
|
||
person_keys = binding.person_field_keys(rows)
|
||
for index, item in enumerate(rows, start=1):
|
||
shown = str(item.get("label") or item.get("key") or "").strip() or "未命名"
|
||
out.extend(_cfg_problems(db, item.get(FIELD_PROP), item,
|
||
f"第 {index} 个字段「{shown}」的外部取数", keys, person_keys))
|
||
if str(item.get("type") or "") != TABLE_TYPE:
|
||
continue
|
||
columns = table_columns(item)
|
||
column_keys = {str(col.get("key") or "").strip() for col in columns}
|
||
column_keys.discard("")
|
||
for column in columns:
|
||
column_shown = str(column.get("label") or column.get("key") or "").strip() or "未命名"
|
||
out.extend(_cfg_problems(
|
||
db, column.get(FIELD_PROP), column,
|
||
f"明细表「{shown}」的列「{column_shown}」的外部取数",
|
||
# 行作用域读得到整张单与本行的列(同名以本行为准),带出只落在这张表的列上
|
||
keys | column_keys, person_keys | binding.person_field_keys(columns),
|
||
target_keys=column_keys, target_noun="列",
|
||
))
|
||
return out
|
||
|
||
|
||
def _cfg_problems(
|
||
db: Session | None, cfg: Any, field: dict, at: str, keys: set[str],
|
||
person_keys: set[str], *, target_keys: set[str] | None = None, target_noun: str = "字段"
|
||
) -> list[str]:
|
||
"""一份 ``fetch`` 配置的毛病:没配就一句都不说,配了就把形状、源、入参、带出判到底。"""
|
||
if cfg is None or cfg == {} or cfg == "":
|
||
return []
|
||
if not isinstance(cfg, dict):
|
||
return [f"{at}的格式不对"]
|
||
out: list[str] = []
|
||
entries = field_sources(cfg)
|
||
if not entries:
|
||
out.append(f"{at}没选数据源")
|
||
rows_cfg = cfg.get("sources")
|
||
if isinstance(rows_cfg, list) and len(rows_cfg) > MAX_FIELD_SOURCES:
|
||
out.append(f"{at}最多挂 {MAX_FIELD_SOURCES} 个数据源,现在配了 {len(rows_cfg)} 个")
|
||
if str(cfg.get("onError") or "") not in ("", "keep", "block", "zero"):
|
||
out.append(f"{at}的失败策略不支持:{cfg.get('onError')}")
|
||
seconds = cfg.get("revalidateAfter")
|
||
if seconds not in (None, "", 0, "0", False):
|
||
try:
|
||
value = int(seconds)
|
||
except (TypeError, ValueError):
|
||
out.append(f"{at}的实时刷新间隔要填整数秒")
|
||
else:
|
||
if value < 1 or value > 86400:
|
||
out.append(f"{at}的实时刷新间隔要在 1 秒到 86400 秒之间(0 = 不主动刷)")
|
||
merge = str(cfg.get("merge") or "first").strip()
|
||
if merge not in MERGE_MODES:
|
||
out.append(f"{at}的合并方式不支持:{merge}")
|
||
if merge in NUMERIC_MERGE_MODES and str(field.get("type") or "") not in MONEY_TYPES:
|
||
out.append(f"{at}要按「{merge}」合并,这一格得是金额或数字类型")
|
||
# 取数这一路要的是「能取出一个值」的源(列表型取不出单值),带出的键还得是源上声明过
|
||
# 的那几个——两句都在内核里。``person_keys`` 是这份清单里的选人字段(明细列那一档连本表
|
||
# 的选人列一起算):拿它才能在设计期就说清"选人字段直接绑进 SQL 的 = 比不中",
|
||
# 而不是等填单的人撞上一条 blocked。
|
||
for at_entry, entry in binding.entry_labels(at, entries):
|
||
out.extend(binding.entry_problems(
|
||
db, entry, keys, at_entry,
|
||
self_key=str(field.get("key") or "").strip(), want=binding.WANT_FETCH,
|
||
person_keys=person_keys, target_keys=target_keys, target_noun=target_noun,
|
||
))
|
||
return out
|