fix(docreader): Excel 百分比被读小 100 倍 —— 加开关(默认关) (#4423)
parent
a7d915a70a
commit
a9290dcb60
11
.env.example
11
.env.example
|
|
@ -425,6 +425,17 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
|
|||
# DOCREADER_DOCX_MAX_PAGES=0
|
||||
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
|
||||
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
|
||||
# ----- XLSX / Excel -----
|
||||
# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
|
||||
# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112
|
||||
# (100 倍偏差)。开启后百分比单元格变成字面量文本 "61.12%"。默认 false:开启会使单元格
|
||||
# 由数值变为字符串,改变下游排序/计算看到的内容;关闭时读取行为与升级前完全一致。
|
||||
# 单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖(true/false)。
|
||||
# DOCREADER_XLSX_PERCENT_AS_TEXT=false
|
||||
# 兜底:同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
|
||||
# 处理(源文件偶有漏刷格式的格子,每命中一次打 INFO 日志)。仅在上一开关开启时生效;
|
||||
# 0=关闭该兜底(默认)。
|
||||
# DOCREADER_XLSX_PCT_ROW_HINT_MIN=0
|
||||
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时用)。
|
||||
# DOCREADER_EXTERNAL_HTTP_PROXY=
|
||||
# DOCREADER_EXTERNAL_HTTPS_PROXY=
|
||||
|
|
|
|||
|
|
@ -463,6 +463,16 @@ services:
|
|||
# ========== 解析通用 ==========
|
||||
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
|
||||
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
|
||||
# ========== XLSX / Excel ==========
|
||||
# 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
|
||||
# 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的
|
||||
# 61.12% 会变成 0.6112(100 倍偏差)。开启后该单元格变成字面量文本 "61.12%"。
|
||||
# 默认 false:开启会使单元格由数值变为字符串,改变下游排序/计算看到的内容,
|
||||
# 存量部署行为保持不变。单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖。
|
||||
- DOCREADER_XLSX_PERCENT_AS_TEXT=${DOCREADER_XLSX_PERCENT_AS_TEXT:-false}
|
||||
# 同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
|
||||
# 处理(源文件偶有漏刷格式的格子)。仅在上一开关开启时生效;0=关闭(默认)。
|
||||
- DOCREADER_XLSX_PCT_ROW_HINT_MIN=${DOCREADER_XLSX_PCT_ROW_HINT_MIN:-0}
|
||||
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时)
|
||||
- DOCREADER_EXTERNAL_HTTP_PROXY=${DOCREADER_EXTERNAL_HTTP_PROXY:-}
|
||||
- DOCREADER_EXTERNAL_HTTPS_PROXY=${DOCREADER_EXTERNAL_HTTPS_PROXY:-}
|
||||
|
|
|
|||
|
|
@ -74,11 +74,16 @@ class ExcelParser(BaseParser):
|
|||
file_name: str = "",
|
||||
file_type: str | None = None,
|
||||
xlsx_first_row_as_header: Any = False,
|
||||
xlsx_percent_as_text: Any = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
|
||||
self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header)
|
||||
|
||||
# Tri-state: None (or an empty value) keeps the deployment default from
|
||||
# DOCREADER_XLSX_PERCENT_AS_TEXT, an explicit value wins. Same
|
||||
# precedence as the pdf_force_scanned parser override.
|
||||
self.xlsx_percent_as_text = _parse_optional_bool(xlsx_percent_as_text)
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse Excel file bytes into a Document object.
|
||||
|
||||
|
|
@ -100,7 +105,11 @@ class ExcelParser(BaseParser):
|
|||
source_blocks: List[dict] = []
|
||||
start, end = 0, 0
|
||||
|
||||
excel_file = _open_excel_file(content, file_type=self.file_type)
|
||||
excel_file = _open_excel_file(
|
||||
content,
|
||||
file_type=self.file_type,
|
||||
percent_as_text=self.xlsx_percent_as_text,
|
||||
)
|
||||
|
||||
# Process each sheet in the Excel file
|
||||
for excel_sheet_name in excel_file.sheet_names:
|
||||
|
|
@ -205,7 +214,19 @@ def _parse_bool(value: Any) -> bool:
|
|||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _prepare_xlsx_bytes(data: bytes) -> bytes:
|
||||
def _parse_optional_bool(value: Any) -> bool | None:
|
||||
"""Parse a tri-state parser override; None keeps the deployment default."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
text = str(value).strip().lower()
|
||||
if not text:
|
||||
return None
|
||||
return text in {"1", "true", "yes", "y", "on"}
|
||||
|
||||
|
||||
def _prepare_xlsx_bytes(data: bytes, percent_as_text: bool | None = None) -> bytes:
|
||||
repaired = repair_xlsx_bytes(data)
|
||||
if repaired is not None:
|
||||
data = repaired
|
||||
|
|
@ -213,7 +234,7 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
|
|||
if readable is not None:
|
||||
data = readable
|
||||
try:
|
||||
return fill_merged_cells_xlsx(data)
|
||||
return fill_merged_cells_xlsx(data, percent_as_text=percent_as_text)
|
||||
except TypeError:
|
||||
# A non-conforming styles.xml (empty/malformed fills) makes openpyxl's
|
||||
# load_workbook raise before pandas ever sees the file (#3637).
|
||||
|
|
@ -222,10 +243,14 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
|
|||
sanitized = sanitize_xlsx_styles(data)
|
||||
if sanitized is None:
|
||||
raise
|
||||
return fill_merged_cells_xlsx(sanitized)
|
||||
return fill_merged_cells_xlsx(sanitized, percent_as_text=percent_as_text)
|
||||
|
||||
|
||||
def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
|
||||
def _open_excel_file(
|
||||
content: bytes,
|
||||
file_type: str | None = None,
|
||||
percent_as_text: bool | None = None,
|
||||
) -> pd.ExcelFile:
|
||||
"""Open an Excel workbook with explicit engine selection and fallbacks."""
|
||||
data = content
|
||||
converted_via_soffice = False
|
||||
|
|
@ -254,7 +279,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
|
|||
|
||||
engine = engine_for_format(ext)
|
||||
if ext == "xlsx":
|
||||
data = _prepare_xlsx_bytes(data)
|
||||
data = _prepare_xlsx_bytes(data, percent_as_text=percent_as_text)
|
||||
engine = "openpyxl"
|
||||
try:
|
||||
return pd.ExcelFile(BytesIO(data), engine=engine)
|
||||
|
|
@ -269,7 +294,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
|
|||
if repaired is None:
|
||||
raise
|
||||
logger.info("Repaired XLSX sharedStrings packaging before parse")
|
||||
data = _prepare_xlsx_bytes(repaired)
|
||||
data = _prepare_xlsx_bytes(repaired, percent_as_text=percent_as_text)
|
||||
continue
|
||||
except ValueError as exc:
|
||||
if converted_via_soffice or "cannot be determined" not in str(exc):
|
||||
|
|
|
|||
|
|
@ -3,18 +3,185 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Percentage number formats (opt-in) --------------------------------------
|
||||
# Excel keeps a percentage as the stored fraction plus a display format: the
|
||||
# cell holds 0.61123366 and "0.00%" renders it as 61.12%. openpyxl and pandas
|
||||
# read only the fraction, so a value the user sees as 61.12% reaches the
|
||||
# pipeline as 0.6112 — a 100x error that a model will copy verbatim into a
|
||||
# report. When this switch is on, percentage cells are materialized as the
|
||||
# literal text the spreadsheet shows ("61.12%") instead.
|
||||
#
|
||||
# It is off by default on purpose: the cell changes from a number into a
|
||||
# string, so anything that sorts, aggregates or computes with it sees different
|
||||
# input. With it off, every workbook is read exactly as before. A single upload
|
||||
# can override it through parser_engine_overrides.xlsx_percent_as_text.
|
||||
PERCENT_AS_TEXT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
|
||||
|
||||
def fill_merged_cells_xlsx(content: bytes) -> bytes:
|
||||
# Only percentages are materialized. Currency and thousands separators change
|
||||
# how a number is displayed, not what it means ("1,234,567" and 1234567 are the
|
||||
# same number), so rewriting them would churn cells for no semantic gain, and
|
||||
# date formats are already converted to datetimes by openpyxl.
|
||||
|
||||
# --- Same-row hint fallback (opt-in, off by default) -------------------------
|
||||
# Workbooks occasionally leave one cell of a percentage row without a
|
||||
# percentage format (a real financial statement had 折旧率 0.0592 formatted as
|
||||
# "0.00_ ;[Red]\-0.00\ "). When a row carries at least this many explicitly
|
||||
# percentage-formatted cells, an unformatted 0 < |value| < 1 number in that row
|
||||
# is treated as a percentage too. Only consulted while percent materialization
|
||||
# is on; 0 disables the fallback.
|
||||
ROW_HINT_MIN_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
|
||||
_DEFAULT_ROW_HINT_MIN = 0
|
||||
|
||||
# Only materialize a known numeric percentage pattern. Excel literals, conditions,
|
||||
# scaling and arbitrary decorations require a full format renderer; leave those
|
||||
# values untouched rather than guessing that every '%' means multiply by 100.
|
||||
_PERCENT_PATTERN = re.compile(
|
||||
r"(?P<sign>-?)(?P<open>\()?(?P<group>#,##)?0"
|
||||
r"(?:\.(?P<decimals>0+))?%(?P<close>\))?"
|
||||
)
|
||||
_FORMAT_COLOR = re.compile(
|
||||
r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color[0-9]+)\]", re.I
|
||||
)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
value = os.environ.get(name)
|
||||
if value is None or not str(value).strip():
|
||||
return default
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
try:
|
||||
return int(str(os.environ.get(name, default)).strip())
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def percent_as_text_default() -> bool:
|
||||
"""Deployment-wide default for materializing percentage formats.
|
||||
|
||||
Read per call (not at import time) so the value can be flipped without a
|
||||
module reload; a per-upload override still wins over it.
|
||||
"""
|
||||
return _env_bool(PERCENT_AS_TEXT_ENV, False)
|
||||
|
||||
|
||||
def _format_percentage(value, number_format: str):
|
||||
"""Render supported Excel percentage formats, leaving other formats intact."""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return None
|
||||
if not number_format or "%" not in number_format:
|
||||
return None
|
||||
# Quoted/escaped percentages are literal text, not percentage operators.
|
||||
# Reject unsupported conditions across all sections before selecting one.
|
||||
cleaned = _FORMAT_COLOR.sub("", number_format)
|
||||
if any(char in cleaned for char in ('"', "\\", "_", "*", "[", "]")):
|
||||
return None
|
||||
sections = cleaned.split(";")
|
||||
if len(sections) > 4:
|
||||
return None
|
||||
section_index = 1 if value < 0 and len(sections) >= 2 else 0
|
||||
if value == 0 and len(sections) >= 3:
|
||||
section_index = 2
|
||||
match = _PERCENT_PATTERN.fullmatch(sections[section_index].strip())
|
||||
if not match or bool(match["open"]) != bool(match["close"]):
|
||||
return None
|
||||
decimals = len(match["decimals"] or "")
|
||||
# Excel's negative section formats the magnitude; a single section instead
|
||||
# supplies the minus sign automatically.
|
||||
amount = abs(value) if section_index == 1 else value
|
||||
grouping = "," if match["group"] else ""
|
||||
try:
|
||||
number = f"{float(amount) * 100:{grouping}.{decimals}f}"
|
||||
return f"{match['sign']}{match['open'] or ''}{number}%{match['close'] or ''}"
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _materialize_percentage_cells(ws, row_hint_min: int) -> tuple[int, int]:
|
||||
"""Rewrite the percentage-formatted numbers of *ws* as their display text.
|
||||
|
||||
Must run before merged ranges are filled: only the master cell of a merge
|
||||
carries a number_format while the covered cells are General, so filling
|
||||
first would copy a bare fraction into rows 2..n and leave it unconverted —
|
||||
the same row would then show both "0.55%" and 0.005509433890047721.
|
||||
|
||||
Returns ``(materialized, hinted)`` cell counts.
|
||||
"""
|
||||
row_hints: dict = {}
|
||||
if row_hint_min > 0:
|
||||
# Count the explicit percentage formats before converting anything:
|
||||
# the conversion below resets number_format to General.
|
||||
for row in ws.iter_rows():
|
||||
count = sum(
|
||||
1
|
||||
for cell in row
|
||||
if type(cell.value) in (int, float) # noqa: E721 - excludes bool
|
||||
and _format_percentage(cell.value, cell.number_format) is not None
|
||||
)
|
||||
if count:
|
||||
row_hints[row[0].row] = count
|
||||
|
||||
materialized = 0
|
||||
hinted = 0
|
||||
for row in ws.iter_rows():
|
||||
hint = row_hints.get(row[0].row, 0)
|
||||
for cell in row:
|
||||
if type(cell.value) not in (int, float): # noqa: E721 - excludes bool
|
||||
continue
|
||||
text = _format_percentage(cell.value, cell.number_format)
|
||||
if (
|
||||
text is None
|
||||
and "%" not in str(cell.number_format or "")
|
||||
and hint >= row_hint_min > 0
|
||||
and 0 < abs(cell.value) < 1
|
||||
):
|
||||
# Fallback for a cell that lost its percentage format while its
|
||||
# row kept one. Logged because it rewrites an unformatted value.
|
||||
text = f"{float(cell.value) * 100:.2f}%"
|
||||
hinted += 1
|
||||
logger.info(
|
||||
"XLSX pre-parse: %s!%s%d has no percentage format but its row "
|
||||
"does; reading it as %.2f%% (stored value %r)",
|
||||
ws.title,
|
||||
cell.column_letter,
|
||||
cell.row,
|
||||
float(cell.value) * 100,
|
||||
cell.value,
|
||||
)
|
||||
if text is None:
|
||||
continue
|
||||
cell.value = text
|
||||
cell.number_format = "General"
|
||||
materialized += 1
|
||||
return materialized, hinted
|
||||
|
||||
|
||||
def fill_merged_cells_xlsx(
|
||||
content: bytes, percent_as_text: bool | None = None
|
||||
) -> bytes:
|
||||
"""Unmerge ranges and copy the master cell value into every covered cell.
|
||||
|
||||
openpyxl only stores values on the top-left cell of a merge; pandas then
|
||||
sees NaN in the rest. Filling makes row-wise RAG chunks retain context.
|
||||
|
||||
``percent_as_text`` also materializes percentage-formatted numbers as the
|
||||
text the spreadsheet displays ("0.6112" -> "61.12%"), because the stored
|
||||
fraction is 100x off from what a reader sees. ``None`` (the default) uses
|
||||
the ``DOCREADER_XLSX_PERCENT_AS_TEXT`` deployment default, which is off, so
|
||||
the default path is unchanged.
|
||||
"""
|
||||
if percent_as_text is None:
|
||||
percent_as_text = percent_as_text_default()
|
||||
|
||||
if not zipfile.is_zipfile(BytesIO(content)):
|
||||
return content
|
||||
|
||||
|
|
@ -22,7 +189,21 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
|
|||
|
||||
wb = load_workbook(BytesIO(content), data_only=True)
|
||||
changed = False
|
||||
materialized_cells = 0
|
||||
hinted_cells = 0
|
||||
row_hint_min = (
|
||||
_env_int(ROW_HINT_MIN_ENV, _DEFAULT_ROW_HINT_MIN) if percent_as_text else 0
|
||||
)
|
||||
|
||||
for ws in wb.worksheets:
|
||||
if percent_as_text:
|
||||
# Before the merge fill below — see _materialize_percentage_cells.
|
||||
materialized, hinted = _materialize_percentage_cells(ws, row_hint_min)
|
||||
materialized_cells += materialized
|
||||
hinted_cells += hinted
|
||||
if materialized:
|
||||
changed = True
|
||||
|
||||
if not ws.merged_cells.ranges:
|
||||
continue
|
||||
for merge_range in list(ws.merged_cells.ranges):
|
||||
|
|
@ -52,5 +233,13 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
|
|||
|
||||
out = BytesIO()
|
||||
wb.save(out)
|
||||
if materialized_cells:
|
||||
logger.info(
|
||||
"Materialized %d percentage cell(s) in XLSX before parse "
|
||||
"(%d recovered by the same-row hint fallback, threshold=%d)",
|
||||
materialized_cells,
|
||||
hinted_cells,
|
||||
row_hint_min,
|
||||
)
|
||||
logger.info("Filled merged cells in XLSX before parse")
|
||||
return out.getvalue()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,377 @@
|
|||
"""Tests for materializing XLSX percentage number formats (opt-in).
|
||||
|
||||
Excel keeps a percentage as the underlying fraction plus a display format:
|
||||
the cell stores ``0.61123366`` and ``0.00%`` renders it as ``61.12%``.
|
||||
openpyxl and pandas read the fraction and drop the format, so a cell the user
|
||||
sees as ``61.12%`` reaches the pipeline as ``0.6112`` — a 100x error that a
|
||||
model will happily copy into a report.
|
||||
|
||||
When ``DOCREADER_XLSX_PERCENT_AS_TEXT`` is on (or the per-upload
|
||||
``parser_engine_overrides`` key ``xlsx_percent_as_text`` is set), percentage
|
||||
cells are materialized as the literal text the user sees. The switch is off by
|
||||
default because the value changes from a number into a string, so anything that
|
||||
sorts or aggregates the cell would see different input; with it off every
|
||||
workbook is read exactly as before.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import io
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import openpyxl
|
||||
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.parser import Parser
|
||||
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx, _format_percentage
|
||||
|
||||
PERCENT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
|
||||
ROW_HINT_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
|
||||
OVERRIDE_KEY = "xlsx_percent_as_text"
|
||||
|
||||
|
||||
def _workbook_with_percentages() -> bytes:
|
||||
"""A real .xlsx mixing percentages, a merge, plain numbers and dates.
|
||||
|
||||
Layout (1-based; ``B3:B4`` is merged, so row 4's B cell is covered and has
|
||||
no format of its own — exactly like the yearly columns of a financial
|
||||
statement, where a label merge covers the row below):
|
||||
|
||||
A B C
|
||||
1 科目 2024 2025
|
||||
2 毛利率 0.61123366 (0.00%) 0.5512345 (0.00%)
|
||||
3 收入占比 0.0833 (0.00%) 0.0912 (0.00%)
|
||||
4 收入占比(续) <covered by B3:B4> 0.005509433890047721 (0.00%)
|
||||
5 营业收入 1234567.89 (#,##0.00) 9876543.21 (#,##0.00)
|
||||
6 报告日 2025-01-31 (yyyy-mm-dd) 2024-12-31 (yyyy-mm-dd)
|
||||
7 汇总 0.5 (General) 6100 (General)
|
||||
|
||||
Row 4 is the regression case: it carries its own percentage cell in C, so a
|
||||
merge fill that runs before the percentage pass leaves that row showing both
|
||||
``0.55%`` and a raw ``0.0833``.
|
||||
"""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "财务"
|
||||
ws.append(["科目", "2024", "2025"])
|
||||
ws.append(["毛利率", 0.61123366, 0.5512345])
|
||||
ws.append(["收入占比", 0.0833, 0.0912])
|
||||
ws.append(["收入占比(续)", None, 0.005509433890047721])
|
||||
ws.append(["营业收入", 1234567.89, 9876543.21])
|
||||
ws.append(["报告日", datetime.date(2025, 1, 31), datetime.date(2024, 12, 31)])
|
||||
ws.append(["汇总", 0.5, 6100])
|
||||
|
||||
for row in (2, 3):
|
||||
ws.cell(row=row, column=2).number_format = "0.00%"
|
||||
ws.cell(row=row, column=3).number_format = "0.00%"
|
||||
ws["C4"].number_format = "0.00%"
|
||||
# Merged after the formats are set: openpyxl keeps only the master (B3)
|
||||
# style, which is what makes the covered cell's format General.
|
||||
ws.merge_cells("B3:B4")
|
||||
for row in (5,):
|
||||
ws.cell(row=row, column=2).number_format = "#,##0.00"
|
||||
ws.cell(row=row, column=3).number_format = "#,##0.00"
|
||||
ws["B6"].number_format = "yyyy-mm-dd"
|
||||
ws["C6"].number_format = "yyyy-mm-dd"
|
||||
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
return bio.getvalue()
|
||||
|
||||
|
||||
def _workbook_with_unformatted_cell_in_percentage_row() -> bytes:
|
||||
"""A percentage row (3 formatted cells) with one cell left General (D2)."""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.append(["项目", "2024", "2025", "2026", "2027"])
|
||||
ws.append(["折旧率", 0.0592, 0.0601, 0.5, 0.0612])
|
||||
for col in (2, 3, 5):
|
||||
ws.cell(row=2, column=col).number_format = "0.00%"
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
return bio.getvalue()
|
||||
|
||||
|
||||
def _workbook_without_merges() -> bytes:
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.append(["项目", "值"])
|
||||
ws.append(["占比", 0.25])
|
||||
ws["B2"].number_format = "0%"
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
return bio.getvalue()
|
||||
|
||||
|
||||
def _read_cells(content: bytes, *coordinates: str):
|
||||
"""Return the values of *coordinates* on the first worksheet."""
|
||||
wb = openpyxl.load_workbook(io.BytesIO(content), data_only=True)
|
||||
ws = wb.worksheets[0]
|
||||
return [ws[c].value for c in coordinates]
|
||||
|
||||
|
||||
def _upstream_fill_merged_cells_xlsx(content: bytes) -> bytes:
|
||||
"""Frozen copy of the pre-change implementation, for byte-identity checks.
|
||||
|
||||
Kept verbatim (docreader/parser/xlsx_merge.py before the percentage switch)
|
||||
so the default path of the new function can be compared byte for byte with
|
||||
the behavior existing deployments have today.
|
||||
"""
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
if not zipfile.is_zipfile(BytesIO(content)):
|
||||
return content
|
||||
|
||||
from openpyxl import load_workbook
|
||||
|
||||
wb = load_workbook(BytesIO(content), data_only=True)
|
||||
changed = False
|
||||
for ws in wb.worksheets:
|
||||
if not ws.merged_cells.ranges:
|
||||
continue
|
||||
for merge_range in list(ws.merged_cells.ranges):
|
||||
master_value = ws.cell(merge_range.min_row, merge_range.min_col).value
|
||||
ws.unmerge_cells(str(merge_range))
|
||||
for row in range(merge_range.min_row, merge_range.max_row + 1):
|
||||
for col in range(merge_range.min_col, merge_range.max_col + 1):
|
||||
ws.cell(row, col).value = master_value
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
return content
|
||||
|
||||
active_sheet = wb.active
|
||||
for chart_sheet in list(wb.chartsheets):
|
||||
wb.remove(chart_sheet)
|
||||
if active_sheet in wb.worksheets:
|
||||
wb.active = active_sheet
|
||||
elif wb.worksheets:
|
||||
wb.active = wb.worksheets[0]
|
||||
for ws in wb.worksheets:
|
||||
ws._charts.clear()
|
||||
|
||||
out = BytesIO()
|
||||
wb.save(out)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
class PercentMaterializationOffTests(unittest.TestCase):
|
||||
"""Default behavior: unchanged, and byte-identical to the old function."""
|
||||
|
||||
def test_default_keeps_the_raw_fraction(self):
|
||||
content = _workbook_with_percentages()
|
||||
|
||||
self.assertEqual(
|
||||
fill_merged_cells_xlsx(content), _upstream_fill_merged_cells_xlsx(content)
|
||||
)
|
||||
|
||||
values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "B4")
|
||||
self.assertEqual(values, [0.61123366, 0.5512345, 0.0833, 0.0833])
|
||||
|
||||
def test_excel_parser_default_keeps_the_raw_fraction(self):
|
||||
document = ExcelParser().parse_into_text(_workbook_with_percentages())
|
||||
|
||||
self.assertIn("A: 毛利率,B: 0.61123366", document.content)
|
||||
self.assertNotIn("61.12%", document.content)
|
||||
|
||||
def test_switch_off_is_byte_identical_to_the_upstream_implementation(self):
|
||||
for build in (
|
||||
_workbook_with_percentages,
|
||||
_workbook_without_merges,
|
||||
_workbook_with_unformatted_cell_in_percentage_row,
|
||||
):
|
||||
content = build()
|
||||
with self.subTest(fixture=build.__name__):
|
||||
# Explicitly off, so a stray env var in the calling shell
|
||||
# cannot turn this into an accidental behavior test.
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "0"}):
|
||||
self.assertEqual(
|
||||
fill_merged_cells_xlsx(content),
|
||||
_upstream_fill_merged_cells_xlsx(content),
|
||||
)
|
||||
|
||||
def test_env_var_off_forms_keep_the_raw_fraction(self):
|
||||
content = _workbook_with_percentages()
|
||||
for raw in ("0", "false", "FALSE", "no", "off", ""):
|
||||
with self.subTest(value=raw):
|
||||
with patch.dict(os.environ, {PERCENT_ENV: raw}):
|
||||
self.assertEqual(
|
||||
_read_cells(fill_merged_cells_xlsx(content), "B2"),
|
||||
[0.61123366],
|
||||
)
|
||||
|
||||
|
||||
class PercentMaterializationOnTests(unittest.TestCase):
|
||||
"""Enabled: percentages become the text the spreadsheet displays."""
|
||||
|
||||
def test_percent_cells_become_display_text(self):
|
||||
content = _workbook_with_percentages()
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "C4")
|
||||
|
||||
self.assertEqual(values, ["61.12%", "55.12%", "8.33%", "0.55%"])
|
||||
|
||||
def test_merged_rows_below_the_master_are_materialized_too(self):
|
||||
"""The order pitfall: B4 is covered by B3:B4 and has no format itself."""
|
||||
content = _workbook_with_percentages()
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
b3, b4 = _read_cells(fill_merged_cells_xlsx(content), "B3", "B4")
|
||||
|
||||
self.assertEqual(b3, "8.33%")
|
||||
self.assertEqual(b4, "8.33%")
|
||||
|
||||
def test_no_row_shows_both_a_percentage_and_a_raw_fraction(self):
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
document = ExcelParser().parse_into_text(_workbook_with_percentages())
|
||||
|
||||
self.assertIn("A: 毛利率,B: 61.12%,C: 55.12%", document.content)
|
||||
self.assertIn("A: 收入占比,B: 8.33%,C: 9.12%", document.content)
|
||||
# Row 4 keeps its own percentage cell (C) and its merged B cell: both
|
||||
# must be text, or the same row would read as two different scales.
|
||||
self.assertIn("A: 收入占比(续),B: 8.33%,C: 0.55%", document.content)
|
||||
for raw in ("0.61123366", "0.5512345", "0.0833", "0.0912", "0.005509433890047721"):
|
||||
self.assertNotIn(raw, document.content)
|
||||
|
||||
def test_currency_plain_numbers_and_dates_are_not_touched(self):
|
||||
content = _workbook_with_percentages()
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
income, income_2025, plain, whole, start, end = _read_cells(
|
||||
fill_merged_cells_xlsx(content), "B5", "C5", "B7", "C7", "B6", "C6"
|
||||
)
|
||||
|
||||
# Thousands separators / currency only affect readability.
|
||||
self.assertEqual(income, 1234567.89)
|
||||
self.assertEqual(income_2025, 9876543.21)
|
||||
# 0.5 has no percentage format, so it must stay a number.
|
||||
self.assertEqual(plain, 0.5)
|
||||
self.assertEqual(whole, 6100)
|
||||
# openpyxl already turns date formats into datetimes.
|
||||
self.assertEqual(start, datetime.datetime(2025, 1, 31))
|
||||
self.assertEqual(end, datetime.datetime(2024, 12, 31))
|
||||
|
||||
def test_decimals_follow_the_number_format(self):
|
||||
"""The text mirrors what the format displays: 0%, 0.0%, 0.00%."""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.append(["项目", "0%", "0.0%", "0.00%", "red"])
|
||||
ws.append(["毛利率", 0.61123366, 0.61123366, 0.61123366, 0.61123366])
|
||||
for col, fmt in (
|
||||
(2, "0%"),
|
||||
(3, "0.0%"),
|
||||
(4, "#,##0.00%"),
|
||||
(5, "0.00%;[Red]-0.00%"),
|
||||
):
|
||||
ws.cell(row=2, column=col).number_format = fmt
|
||||
bio = io.BytesIO()
|
||||
wb.save(bio)
|
||||
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
values = _read_cells(
|
||||
fill_merged_cells_xlsx(bio.getvalue()), "B2", "C2", "D2", "E2"
|
||||
)
|
||||
|
||||
self.assertEqual(values, ["61%", "61.1%", "61.12%", "61.12%"])
|
||||
|
||||
def test_unformatted_cell_in_a_percentage_row_needs_the_opt_in_fallback(self):
|
||||
content = _workbook_with_unformatted_cell_in_percentage_row()
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
self.assertEqual(
|
||||
_read_cells(fill_merged_cells_xlsx(content), "D2"), [0.5]
|
||||
)
|
||||
with patch.dict(
|
||||
os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "3"}
|
||||
):
|
||||
self.assertEqual(
|
||||
_read_cells(fill_merged_cells_xlsx(content), "D2"), ["50.00%"]
|
||||
)
|
||||
|
||||
|
||||
class PercentageFormatSyntaxTests(unittest.TestCase):
|
||||
def test_literal_and_conditional_percent_formats_are_not_scaled(self):
|
||||
for fmt in (r"0.00\%", '0.00"%"', '[>1]0%;0.00%', '0.00%" growth"'):
|
||||
with self.subTest(fmt=fmt):
|
||||
self.assertIsNone(_format_percentage(0.5, fmt))
|
||||
|
||||
def test_selects_the_section_for_the_value(self):
|
||||
for value, fmt, expected in (
|
||||
(0.5, "0%;0.00%", "50%"),
|
||||
(-0.5, "0%;[Red]-0.00%", "-50.00%"),
|
||||
(-0.5, "0%;(0.00%)", "(50.00%)"),
|
||||
(0, "0%;0.00%;0.0%", "0.0%"),
|
||||
(-0.5, "0.00%", "-50.00%"),
|
||||
(12.345, "#,##0.00%", "1,234.50%"),
|
||||
):
|
||||
with self.subTest(value=value, fmt=fmt):
|
||||
self.assertEqual(_format_percentage(value, fmt), expected)
|
||||
|
||||
def test_row_hint_does_not_convert_literal_percent_or_count_it_as_a_hint(self):
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.append(["label", "percent", "literal", "plain"])
|
||||
ws.append(["mixed", 0.5, 0.5, 0.5])
|
||||
ws.append(["literal only", 0.5, 0.5, 0.5])
|
||||
ws["B2"].number_format = "0%"
|
||||
ws["C2"].number_format = r"0.00\%"
|
||||
ws["B3"].number_format = '0.00"%"'
|
||||
data = io.BytesIO()
|
||||
wb.save(data)
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "1"}):
|
||||
result = fill_merged_cells_xlsx(data.getvalue())
|
||||
self.assertEqual(_read_cells(result, "B2", "C2", "D2"), ["50%", 0.5, "50.00%"])
|
||||
self.assertEqual(_read_cells(result, "B3", "C3", "D3"), [0.5, 0.5, 0.5])
|
||||
|
||||
|
||||
class PercentMaterializationSwitchTests(unittest.TestCase):
|
||||
"""The switch itself: env default, per-upload override, precedence."""
|
||||
|
||||
def test_env_var_true_forms_enable_materialization(self):
|
||||
content = _workbook_with_percentages()
|
||||
for raw in ("1", "true", "TRUE", "yes", "y", "on"):
|
||||
with self.subTest(value=raw):
|
||||
with patch.dict(os.environ, {PERCENT_ENV: raw}):
|
||||
self.assertEqual(
|
||||
_read_cells(fill_merged_cells_xlsx(content), "B2"),
|
||||
["61.12%"],
|
||||
)
|
||||
|
||||
def test_excel_parser_accepts_the_override_key(self):
|
||||
document = ExcelParser(xlsx_percent_as_text="true").parse_into_text(
|
||||
_workbook_with_percentages()
|
||||
)
|
||||
|
||||
self.assertIn("A: 毛利率,B: 61.12%", document.content)
|
||||
|
||||
def test_parser_engine_overrides_key_enables_materialization(self):
|
||||
document = Parser().parse_file(
|
||||
"report.xlsx",
|
||||
"xlsx",
|
||||
_workbook_with_percentages(),
|
||||
engine_overrides={OVERRIDE_KEY: "true"},
|
||||
)
|
||||
|
||||
self.assertIn("A: 毛利率,B: 61.12%", document.content)
|
||||
|
||||
def test_upload_override_wins_over_the_env_default(self):
|
||||
content = _workbook_with_percentages()
|
||||
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
off = ExcelParser(xlsx_percent_as_text="false").parse_into_text(content)
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "false"}):
|
||||
on = ExcelParser(xlsx_percent_as_text="true").parse_into_text(content)
|
||||
|
||||
self.assertNotIn("61.12%", off.content)
|
||||
self.assertIn("B: 0.61123366", off.content)
|
||||
self.assertIn("61.12%", on.content)
|
||||
|
||||
def test_absent_override_keeps_the_env_default(self):
|
||||
content = _workbook_with_percentages()
|
||||
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
|
||||
document = ExcelParser().parse_into_text(content)
|
||||
|
||||
self.assertIn("A: 毛利率,B: 61.12%", document.content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -202,10 +202,16 @@ sequenceDiagram
|
|||
三个辅助模块解决现实中的脏文件:
|
||||
|
||||
- **`excel_convert.py`**:魔数/`inspect_excel_format` 检测真实格式(xlsx/xls/xlsb/ods),为每种格式选 pandas engine(`xlrd`/`openpyxl`/`odf`);无法识别时(如 WPS `.et`、被改名的 csv)用 LibreOffice `convert-to xlsx` 归一化(`normalize_excel_bytes` 依次尝试 `.xlsx/.xls/.et/.csv` 后缀)。
|
||||
- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。
|
||||
- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。可选地**按显示格式物化百分比**(见下)。
|
||||
- **`xlsx_repair.py`**:修复常见 XLSX 打包问题——`sharedStrings.xml` 路径大小写/位置不规范时重命名归位;manifest 引用了 sharedStrings 但包内缺失且工作表只用 inline string 时,从 `[Content_Types].xml` 与 workbook rels 中剥掉引用,使 openpyxl 能读。
|
||||
|
||||
XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。
|
||||
**百分比格式**:Excel 的百分比是**显示格式而非存储值**——单元格实存 `0.61123366`,靠 `0.00%` 渲染成 `61.12%`,而 pandas/openpyxl 读的是原始值,格式被丢掉,于是 `61.12%` 变成 `0.6112`(**100 倍偏差**,财务报表类数据会直接把错数字喂给模型)。打开 `DOCREADER_XLSX_PERCENT_AS_TEXT`(或单次上传给 `parser_engine_overrides.xlsx_percent_as_text=true`,优先级高于环境变量)后,带百分比格式的数值单元格会被就地转成显示文本 `"61.12%"`(格式置 `General`),转换在**填合并单元格之前**完成——只有合并区域左上角带 `number_format`,被覆盖的格子是 `General`,先填合并会让第 2..n 行漏转。**只处理百分比**:货币/千分位只影响可读性(`1,234,567` 和 `1234567` 是同一个数),日期由 openpyxl 自动转 datetime。默认 **false**——开启后该单元格由数值变成字符串,会影响下游排序/计算,关闭时读取行为与升级前完全一致。
|
||||
|
||||
格式识别仅转换可可靠渲染的数字百分比模式(含颜色及正数/负数/零分段)。转义或引号中的 `%` 是字面量,不会触发乘 100;条件、填充和其他复杂自定义格式保留原始值,避免错误缩放。字面量百分号单元格也不会触发行提示推断。
|
||||
|
||||
同源文件偶尔有「一格忘了刷百分比格式」的单元格(如某财报 `D12` 折旧率 `0.0592` 格式为 `0.00_ ;[Red]\-0.00\ `)。把 `DOCREADER_XLSX_PCT_ROW_HINT_MIN` 设为 N(默认 **0=关闭**)后,同一行有 ≥N 个显式百分比格式单元格时,该行漏标的 `0<|v|<1` 数字也按百分比处理,每次命中打 INFO 日志可事后审计。
|
||||
|
||||
XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。百分比物化(若开启)发生在 `fill_merged_cells` 内部、填合并单元格之前。
|
||||
|
||||
#### ppt_convert.py / pptx_media.py(.ppt / .pptx,服务于 markitdown 引擎) {#_3-5-ppt-convert-py-pptx-media-py-ppt-pptx-服务于-markitdown-引擎}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue