fix(docreader): Excel 百分比被读小 100 倍 —— 加开关(默认关) (#4423)

main
linus-liu-web 2026-10-17 23:16:45 +08:00 committed by GitHub
parent a7d915a70a
commit a9290dcb60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 629 additions and 11 deletions

View File

@ -425,6 +425,17 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# DOCREADER_DOCX_MAX_PAGES=0
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
# ----- XLSX / Excel -----
# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112
# (100 倍偏差)。开启后百分比单元格变成字面量文本 "61.12%"。默认 false:开启会使单元格
# 由数值变为字符串,改变下游排序/计算看到的内容;关闭时读取行为与升级前完全一致。
# 单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖(true/false)。
# DOCREADER_XLSX_PERCENT_AS_TEXT=false
# 兜底:同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
# 处理(源文件偶有漏刷格式的格子,每命中一次打 INFO 日志)。仅在上一开关开启时生效;
# 0=关闭该兜底(默认)。
# DOCREADER_XLSX_PCT_ROW_HINT_MIN=0
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时用)。
# DOCREADER_EXTERNAL_HTTP_PROXY=
# DOCREADER_EXTERNAL_HTTPS_PROXY=

View File

@ -463,6 +463,16 @@ services:
# ========== 解析通用 ==========
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
# ========== XLSX / Excel ==========
# 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
# 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的
# 61.12% 会变成 0.6112(100 倍偏差)。开启后该单元格变成字面量文本 "61.12%"。
# 默认 false:开启会使单元格由数值变为字符串,改变下游排序/计算看到的内容,
# 存量部署行为保持不变。单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖。
- DOCREADER_XLSX_PERCENT_AS_TEXT=${DOCREADER_XLSX_PERCENT_AS_TEXT:-false}
# 同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
# 处理(源文件偶有漏刷格式的格子)。仅在上一开关开启时生效;0=关闭(默认)。
- DOCREADER_XLSX_PCT_ROW_HINT_MIN=${DOCREADER_XLSX_PCT_ROW_HINT_MIN:-0}
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时)
- DOCREADER_EXTERNAL_HTTP_PROXY=${DOCREADER_EXTERNAL_HTTP_PROXY:-}
- DOCREADER_EXTERNAL_HTTPS_PROXY=${DOCREADER_EXTERNAL_HTTPS_PROXY:-}

View File

@ -74,11 +74,16 @@ class ExcelParser(BaseParser):
file_name: str = "",
file_type: str | None = None,
xlsx_first_row_as_header: Any = False,
xlsx_percent_as_text: Any = None,
**kwargs: Any,
):
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header)
# Tri-state: None (or an empty value) keeps the deployment default from
# DOCREADER_XLSX_PERCENT_AS_TEXT, an explicit value wins. Same
# precedence as the pdf_force_scanned parser override.
self.xlsx_percent_as_text = _parse_optional_bool(xlsx_percent_as_text)
def parse_into_text(self, content: bytes) -> Document:
"""Parse Excel file bytes into a Document object.
@ -100,7 +105,11 @@ class ExcelParser(BaseParser):
source_blocks: List[dict] = []
start, end = 0, 0
excel_file = _open_excel_file(content, file_type=self.file_type)
excel_file = _open_excel_file(
content,
file_type=self.file_type,
percent_as_text=self.xlsx_percent_as_text,
)
# Process each sheet in the Excel file
for excel_sheet_name in excel_file.sheet_names:
@ -205,7 +214,19 @@ def _parse_bool(value: Any) -> bool:
return str(value).strip().lower() in {"1", "true", "yes", "on"}
def _prepare_xlsx_bytes(data: bytes) -> bytes:
def _parse_optional_bool(value: Any) -> bool | None:
"""Parse a tri-state parser override; None keeps the deployment default."""
if value is None:
return None
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if not text:
return None
return text in {"1", "true", "yes", "y", "on"}
def _prepare_xlsx_bytes(data: bytes, percent_as_text: bool | None = None) -> bytes:
repaired = repair_xlsx_bytes(data)
if repaired is not None:
data = repaired
@ -213,7 +234,7 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
if readable is not None:
data = readable
try:
return fill_merged_cells_xlsx(data)
return fill_merged_cells_xlsx(data, percent_as_text=percent_as_text)
except TypeError:
# A non-conforming styles.xml (empty/malformed fills) makes openpyxl's
# load_workbook raise before pandas ever sees the file (#3637).
@ -222,10 +243,14 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
sanitized = sanitize_xlsx_styles(data)
if sanitized is None:
raise
return fill_merged_cells_xlsx(sanitized)
return fill_merged_cells_xlsx(sanitized, percent_as_text=percent_as_text)
def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
def _open_excel_file(
content: bytes,
file_type: str | None = None,
percent_as_text: bool | None = None,
) -> pd.ExcelFile:
"""Open an Excel workbook with explicit engine selection and fallbacks."""
data = content
converted_via_soffice = False
@ -254,7 +279,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
engine = engine_for_format(ext)
if ext == "xlsx":
data = _prepare_xlsx_bytes(data)
data = _prepare_xlsx_bytes(data, percent_as_text=percent_as_text)
engine = "openpyxl"
try:
return pd.ExcelFile(BytesIO(data), engine=engine)
@ -269,7 +294,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
if repaired is None:
raise
logger.info("Repaired XLSX sharedStrings packaging before parse")
data = _prepare_xlsx_bytes(repaired)
data = _prepare_xlsx_bytes(repaired, percent_as_text=percent_as_text)
continue
except ValueError as exc:
if converted_via_soffice or "cannot be determined" not in str(exc):

View File

@ -3,18 +3,185 @@
from __future__ import annotations
import logging
import os
import re
import zipfile
from io import BytesIO
logger = logging.getLogger(__name__)
# --- Percentage number formats (opt-in) --------------------------------------
# Excel keeps a percentage as the stored fraction plus a display format: the
# cell holds 0.61123366 and "0.00%" renders it as 61.12%. openpyxl and pandas
# read only the fraction, so a value the user sees as 61.12% reaches the
# pipeline as 0.6112 — a 100x error that a model will copy verbatim into a
# report. When this switch is on, percentage cells are materialized as the
# literal text the spreadsheet shows ("61.12%") instead.
#
# It is off by default on purpose: the cell changes from a number into a
# string, so anything that sorts, aggregates or computes with it sees different
# input. With it off, every workbook is read exactly as before. A single upload
# can override it through parser_engine_overrides.xlsx_percent_as_text.
PERCENT_AS_TEXT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
def fill_merged_cells_xlsx(content: bytes) -> bytes:
# Only percentages are materialized. Currency and thousands separators change
# how a number is displayed, not what it means ("1,234,567" and 1234567 are the
# same number), so rewriting them would churn cells for no semantic gain, and
# date formats are already converted to datetimes by openpyxl.
# --- Same-row hint fallback (opt-in, off by default) -------------------------
# Workbooks occasionally leave one cell of a percentage row without a
# percentage format (a real financial statement had 折旧率 0.0592 formatted as
# "0.00_ ;[Red]\-0.00\ "). When a row carries at least this many explicitly
# percentage-formatted cells, an unformatted 0 < |value| < 1 number in that row
# is treated as a percentage too. Only consulted while percent materialization
# is on; 0 disables the fallback.
ROW_HINT_MIN_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
_DEFAULT_ROW_HINT_MIN = 0
# Only materialize a known numeric percentage pattern. Excel literals, conditions,
# scaling and arbitrary decorations require a full format renderer; leave those
# values untouched rather than guessing that every '%' means multiply by 100.
_PERCENT_PATTERN = re.compile(
r"(?P<sign>-?)(?P<open>\()?(?P<group>#,##)?0"
r"(?:\.(?P<decimals>0+))?%(?P<close>\))?"
)
_FORMAT_COLOR = re.compile(
r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color[0-9]+)\]", re.I
)
def _env_bool(name: str, default: bool) -> bool:
value = os.environ.get(name)
if value is None or not str(value).strip():
return default
return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
def _env_int(name: str, default: int) -> int:
try:
return int(str(os.environ.get(name, default)).strip())
except (TypeError, ValueError):
return default
def percent_as_text_default() -> bool:
"""Deployment-wide default for materializing percentage formats.
Read per call (not at import time) so the value can be flipped without a
module reload; a per-upload override still wins over it.
"""
return _env_bool(PERCENT_AS_TEXT_ENV, False)
def _format_percentage(value, number_format: str):
"""Render supported Excel percentage formats, leaving other formats intact."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
if not number_format or "%" not in number_format:
return None
# Quoted/escaped percentages are literal text, not percentage operators.
# Reject unsupported conditions across all sections before selecting one.
cleaned = _FORMAT_COLOR.sub("", number_format)
if any(char in cleaned for char in ('"', "\\", "_", "*", "[", "]")):
return None
sections = cleaned.split(";")
if len(sections) > 4:
return None
section_index = 1 if value < 0 and len(sections) >= 2 else 0
if value == 0 and len(sections) >= 3:
section_index = 2
match = _PERCENT_PATTERN.fullmatch(sections[section_index].strip())
if not match or bool(match["open"]) != bool(match["close"]):
return None
decimals = len(match["decimals"] or "")
# Excel's negative section formats the magnitude; a single section instead
# supplies the minus sign automatically.
amount = abs(value) if section_index == 1 else value
grouping = "," if match["group"] else ""
try:
number = f"{float(amount) * 100:{grouping}.{decimals}f}"
return f"{match['sign']}{match['open'] or ''}{number}%{match['close'] or ''}"
except (TypeError, ValueError, OverflowError):
return None
def _materialize_percentage_cells(ws, row_hint_min: int) -> tuple[int, int]:
"""Rewrite the percentage-formatted numbers of *ws* as their display text.
Must run before merged ranges are filled: only the master cell of a merge
carries a number_format while the covered cells are General, so filling
first would copy a bare fraction into rows 2..n and leave it unconverted —
the same row would then show both "0.55%" and 0.005509433890047721.
Returns ``(materialized, hinted)`` cell counts.
"""
row_hints: dict = {}
if row_hint_min > 0:
# Count the explicit percentage formats before converting anything:
# the conversion below resets number_format to General.
for row in ws.iter_rows():
count = sum(
1
for cell in row
if type(cell.value) in (int, float) # noqa: E721 - excludes bool
and _format_percentage(cell.value, cell.number_format) is not None
)
if count:
row_hints[row[0].row] = count
materialized = 0
hinted = 0
for row in ws.iter_rows():
hint = row_hints.get(row[0].row, 0)
for cell in row:
if type(cell.value) not in (int, float): # noqa: E721 - excludes bool
continue
text = _format_percentage(cell.value, cell.number_format)
if (
text is None
and "%" not in str(cell.number_format or "")
and hint >= row_hint_min > 0
and 0 < abs(cell.value) < 1
):
# Fallback for a cell that lost its percentage format while its
# row kept one. Logged because it rewrites an unformatted value.
text = f"{float(cell.value) * 100:.2f}%"
hinted += 1
logger.info(
"XLSX pre-parse: %s!%s%d has no percentage format but its row "
"does; reading it as %.2f%% (stored value %r)",
ws.title,
cell.column_letter,
cell.row,
float(cell.value) * 100,
cell.value,
)
if text is None:
continue
cell.value = text
cell.number_format = "General"
materialized += 1
return materialized, hinted
def fill_merged_cells_xlsx(
content: bytes, percent_as_text: bool | None = None
) -> bytes:
"""Unmerge ranges and copy the master cell value into every covered cell.
openpyxl only stores values on the top-left cell of a merge; pandas then
sees NaN in the rest. Filling makes row-wise RAG chunks retain context.
``percent_as_text`` also materializes percentage-formatted numbers as the
text the spreadsheet displays ("0.6112" -> "61.12%"), because the stored
fraction is 100x off from what a reader sees. ``None`` (the default) uses
the ``DOCREADER_XLSX_PERCENT_AS_TEXT`` deployment default, which is off, so
the default path is unchanged.
"""
if percent_as_text is None:
percent_as_text = percent_as_text_default()
if not zipfile.is_zipfile(BytesIO(content)):
return content
@ -22,7 +189,21 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
wb = load_workbook(BytesIO(content), data_only=True)
changed = False
materialized_cells = 0
hinted_cells = 0
row_hint_min = (
_env_int(ROW_HINT_MIN_ENV, _DEFAULT_ROW_HINT_MIN) if percent_as_text else 0
)
for ws in wb.worksheets:
if percent_as_text:
# Before the merge fill below — see _materialize_percentage_cells.
materialized, hinted = _materialize_percentage_cells(ws, row_hint_min)
materialized_cells += materialized
hinted_cells += hinted
if materialized:
changed = True
if not ws.merged_cells.ranges:
continue
for merge_range in list(ws.merged_cells.ranges):
@ -52,5 +233,13 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
out = BytesIO()
wb.save(out)
if materialized_cells:
logger.info(
"Materialized %d percentage cell(s) in XLSX before parse "
"(%d recovered by the same-row hint fallback, threshold=%d)",
materialized_cells,
hinted_cells,
row_hint_min,
)
logger.info("Filled merged cells in XLSX before parse")
return out.getvalue()

View File

@ -0,0 +1,377 @@
"""Tests for materializing XLSX percentage number formats (opt-in).
Excel keeps a percentage as the underlying fraction plus a display format:
the cell stores ``0.61123366`` and ``0.00%`` renders it as ``61.12%``.
openpyxl and pandas read the fraction and drop the format, so a cell the user
sees as ``61.12%`` reaches the pipeline as ``0.6112`` — a 100x error that a
model will happily copy into a report.
When ``DOCREADER_XLSX_PERCENT_AS_TEXT`` is on (or the per-upload
``parser_engine_overrides`` key ``xlsx_percent_as_text`` is set), percentage
cells are materialized as the literal text the user sees. The switch is off by
default because the value changes from a number into a string, so anything that
sorts or aggregates the cell would see different input; with it off every
workbook is read exactly as before.
"""
import datetime
import io
import os
import unittest
from unittest.mock import patch
import openpyxl
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.parser import Parser
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx, _format_percentage
PERCENT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
ROW_HINT_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
OVERRIDE_KEY = "xlsx_percent_as_text"
def _workbook_with_percentages() -> bytes:
"""A real .xlsx mixing percentages, a merge, plain numbers and dates.
Layout (1-based; ``B3:B4`` is merged, so row 4's B cell is covered and has
no format of its own — exactly like the yearly columns of a financial
statement, where a label merge covers the row below):
A B C
1 科目 2024 2025
2 毛利率 0.61123366 (0.00%) 0.5512345 (0.00%)
3 收入占比 0.0833 (0.00%) 0.0912 (0.00%)
4 收入占比(续) <covered by B3:B4> 0.005509433890047721 (0.00%)
5 营业收入 1234567.89 (#,##0.00) 9876543.21 (#,##0.00)
6 报告日 2025-01-31 (yyyy-mm-dd) 2024-12-31 (yyyy-mm-dd)
7 汇总 0.5 (General) 6100 (General)
Row 4 is the regression case: it carries its own percentage cell in C, so a
merge fill that runs before the percentage pass leaves that row showing both
``0.55%`` and a raw ``0.0833``.
"""
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "财务"
ws.append(["科目", "2024", "2025"])
ws.append(["毛利率", 0.61123366, 0.5512345])
ws.append(["收入占比", 0.0833, 0.0912])
ws.append(["收入占比(续)", None, 0.005509433890047721])
ws.append(["营业收入", 1234567.89, 9876543.21])
ws.append(["报告日", datetime.date(2025, 1, 31), datetime.date(2024, 12, 31)])
ws.append(["汇总", 0.5, 6100])
for row in (2, 3):
ws.cell(row=row, column=2).number_format = "0.00%"
ws.cell(row=row, column=3).number_format = "0.00%"
ws["C4"].number_format = "0.00%"
# Merged after the formats are set: openpyxl keeps only the master (B3)
# style, which is what makes the covered cell's format General.
ws.merge_cells("B3:B4")
for row in (5,):
ws.cell(row=row, column=2).number_format = "#,##0.00"
ws.cell(row=row, column=3).number_format = "#,##0.00"
ws["B6"].number_format = "yyyy-mm-dd"
ws["C6"].number_format = "yyyy-mm-dd"
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def _workbook_with_unformatted_cell_in_percentage_row() -> bytes:
"""A percentage row (3 formatted cells) with one cell left General (D2)."""
wb = openpyxl.Workbook()
ws = wb.active
ws.append(["项目", "2024", "2025", "2026", "2027"])
ws.append(["折旧率", 0.0592, 0.0601, 0.5, 0.0612])
for col in (2, 3, 5):
ws.cell(row=2, column=col).number_format = "0.00%"
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def _workbook_without_merges() -> bytes:
wb = openpyxl.Workbook()
ws = wb.active
ws.append(["项目", "值"])
ws.append(["占比", 0.25])
ws["B2"].number_format = "0%"
bio = io.BytesIO()
wb.save(bio)
return bio.getvalue()
def _read_cells(content: bytes, *coordinates: str):
"""Return the values of *coordinates* on the first worksheet."""
wb = openpyxl.load_workbook(io.BytesIO(content), data_only=True)
ws = wb.worksheets[0]
return [ws[c].value for c in coordinates]
def _upstream_fill_merged_cells_xlsx(content: bytes) -> bytes:
"""Frozen copy of the pre-change implementation, for byte-identity checks.
Kept verbatim (docreader/parser/xlsx_merge.py before the percentage switch)
so the default path of the new function can be compared byte for byte with
the behavior existing deployments have today.
"""
import zipfile
from io import BytesIO
if not zipfile.is_zipfile(BytesIO(content)):
return content
from openpyxl import load_workbook
wb = load_workbook(BytesIO(content), data_only=True)
changed = False
for ws in wb.worksheets:
if not ws.merged_cells.ranges:
continue
for merge_range in list(ws.merged_cells.ranges):
master_value = ws.cell(merge_range.min_row, merge_range.min_col).value
ws.unmerge_cells(str(merge_range))
for row in range(merge_range.min_row, merge_range.max_row + 1):
for col in range(merge_range.min_col, merge_range.max_col + 1):
ws.cell(row, col).value = master_value
changed = True
if not changed:
return content
active_sheet = wb.active
for chart_sheet in list(wb.chartsheets):
wb.remove(chart_sheet)
if active_sheet in wb.worksheets:
wb.active = active_sheet
elif wb.worksheets:
wb.active = wb.worksheets[0]
for ws in wb.worksheets:
ws._charts.clear()
out = BytesIO()
wb.save(out)
return out.getvalue()
class PercentMaterializationOffTests(unittest.TestCase):
"""Default behavior: unchanged, and byte-identical to the old function."""
def test_default_keeps_the_raw_fraction(self):
content = _workbook_with_percentages()
self.assertEqual(
fill_merged_cells_xlsx(content), _upstream_fill_merged_cells_xlsx(content)
)
values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "B4")
self.assertEqual(values, [0.61123366, 0.5512345, 0.0833, 0.0833])
def test_excel_parser_default_keeps_the_raw_fraction(self):
document = ExcelParser().parse_into_text(_workbook_with_percentages())
self.assertIn("A: 毛利率,B: 0.61123366", document.content)
self.assertNotIn("61.12%", document.content)
def test_switch_off_is_byte_identical_to_the_upstream_implementation(self):
for build in (
_workbook_with_percentages,
_workbook_without_merges,
_workbook_with_unformatted_cell_in_percentage_row,
):
content = build()
with self.subTest(fixture=build.__name__):
# Explicitly off, so a stray env var in the calling shell
# cannot turn this into an accidental behavior test.
with patch.dict(os.environ, {PERCENT_ENV: "0"}):
self.assertEqual(
fill_merged_cells_xlsx(content),
_upstream_fill_merged_cells_xlsx(content),
)
def test_env_var_off_forms_keep_the_raw_fraction(self):
content = _workbook_with_percentages()
for raw in ("0", "false", "FALSE", "no", "off", ""):
with self.subTest(value=raw):
with patch.dict(os.environ, {PERCENT_ENV: raw}):
self.assertEqual(
_read_cells(fill_merged_cells_xlsx(content), "B2"),
[0.61123366],
)
class PercentMaterializationOnTests(unittest.TestCase):
"""Enabled: percentages become the text the spreadsheet displays."""
def test_percent_cells_become_display_text(self):
content = _workbook_with_percentages()
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "C4")
self.assertEqual(values, ["61.12%", "55.12%", "8.33%", "0.55%"])
def test_merged_rows_below_the_master_are_materialized_too(self):
"""The order pitfall: B4 is covered by B3:B4 and has no format itself."""
content = _workbook_with_percentages()
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
b3, b4 = _read_cells(fill_merged_cells_xlsx(content), "B3", "B4")
self.assertEqual(b3, "8.33%")
self.assertEqual(b4, "8.33%")
def test_no_row_shows_both_a_percentage_and_a_raw_fraction(self):
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
document = ExcelParser().parse_into_text(_workbook_with_percentages())
self.assertIn("A: 毛利率,B: 61.12%,C: 55.12%", document.content)
self.assertIn("A: 收入占比,B: 8.33%,C: 9.12%", document.content)
# Row 4 keeps its own percentage cell (C) and its merged B cell: both
# must be text, or the same row would read as two different scales.
self.assertIn("A: 收入占比(续),B: 8.33%,C: 0.55%", document.content)
for raw in ("0.61123366", "0.5512345", "0.0833", "0.0912", "0.005509433890047721"):
self.assertNotIn(raw, document.content)
def test_currency_plain_numbers_and_dates_are_not_touched(self):
content = _workbook_with_percentages()
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
income, income_2025, plain, whole, start, end = _read_cells(
fill_merged_cells_xlsx(content), "B5", "C5", "B7", "C7", "B6", "C6"
)
# Thousands separators / currency only affect readability.
self.assertEqual(income, 1234567.89)
self.assertEqual(income_2025, 9876543.21)
# 0.5 has no percentage format, so it must stay a number.
self.assertEqual(plain, 0.5)
self.assertEqual(whole, 6100)
# openpyxl already turns date formats into datetimes.
self.assertEqual(start, datetime.datetime(2025, 1, 31))
self.assertEqual(end, datetime.datetime(2024, 12, 31))
def test_decimals_follow_the_number_format(self):
"""The text mirrors what the format displays: 0%, 0.0%, 0.00%."""
wb = openpyxl.Workbook()
ws = wb.active
ws.append(["项目", "0%", "0.0%", "0.00%", "red"])
ws.append(["毛利率", 0.61123366, 0.61123366, 0.61123366, 0.61123366])
for col, fmt in (
(2, "0%"),
(3, "0.0%"),
(4, "#,##0.00%"),
(5, "0.00%;[Red]-0.00%"),
):
ws.cell(row=2, column=col).number_format = fmt
bio = io.BytesIO()
wb.save(bio)
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
values = _read_cells(
fill_merged_cells_xlsx(bio.getvalue()), "B2", "C2", "D2", "E2"
)
self.assertEqual(values, ["61%", "61.1%", "61.12%", "61.12%"])
def test_unformatted_cell_in_a_percentage_row_needs_the_opt_in_fallback(self):
content = _workbook_with_unformatted_cell_in_percentage_row()
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
self.assertEqual(
_read_cells(fill_merged_cells_xlsx(content), "D2"), [0.5]
)
with patch.dict(
os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "3"}
):
self.assertEqual(
_read_cells(fill_merged_cells_xlsx(content), "D2"), ["50.00%"]
)
class PercentageFormatSyntaxTests(unittest.TestCase):
def test_literal_and_conditional_percent_formats_are_not_scaled(self):
for fmt in (r"0.00\%", '0.00"%"', '[>1]0%;0.00%', '0.00%" growth"'):
with self.subTest(fmt=fmt):
self.assertIsNone(_format_percentage(0.5, fmt))
def test_selects_the_section_for_the_value(self):
for value, fmt, expected in (
(0.5, "0%;0.00%", "50%"),
(-0.5, "0%;[Red]-0.00%", "-50.00%"),
(-0.5, "0%;(0.00%)", "(50.00%)"),
(0, "0%;0.00%;0.0%", "0.0%"),
(-0.5, "0.00%", "-50.00%"),
(12.345, "#,##0.00%", "1,234.50%"),
):
with self.subTest(value=value, fmt=fmt):
self.assertEqual(_format_percentage(value, fmt), expected)
def test_row_hint_does_not_convert_literal_percent_or_count_it_as_a_hint(self):
wb = openpyxl.Workbook()
ws = wb.active
ws.append(["label", "percent", "literal", "plain"])
ws.append(["mixed", 0.5, 0.5, 0.5])
ws.append(["literal only", 0.5, 0.5, 0.5])
ws["B2"].number_format = "0%"
ws["C2"].number_format = r"0.00\%"
ws["B3"].number_format = '0.00"%"'
data = io.BytesIO()
wb.save(data)
with patch.dict(os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "1"}):
result = fill_merged_cells_xlsx(data.getvalue())
self.assertEqual(_read_cells(result, "B2", "C2", "D2"), ["50%", 0.5, "50.00%"])
self.assertEqual(_read_cells(result, "B3", "C3", "D3"), [0.5, 0.5, 0.5])
class PercentMaterializationSwitchTests(unittest.TestCase):
"""The switch itself: env default, per-upload override, precedence."""
def test_env_var_true_forms_enable_materialization(self):
content = _workbook_with_percentages()
for raw in ("1", "true", "TRUE", "yes", "y", "on"):
with self.subTest(value=raw):
with patch.dict(os.environ, {PERCENT_ENV: raw}):
self.assertEqual(
_read_cells(fill_merged_cells_xlsx(content), "B2"),
["61.12%"],
)
def test_excel_parser_accepts_the_override_key(self):
document = ExcelParser(xlsx_percent_as_text="true").parse_into_text(
_workbook_with_percentages()
)
self.assertIn("A: 毛利率,B: 61.12%", document.content)
def test_parser_engine_overrides_key_enables_materialization(self):
document = Parser().parse_file(
"report.xlsx",
"xlsx",
_workbook_with_percentages(),
engine_overrides={OVERRIDE_KEY: "true"},
)
self.assertIn("A: 毛利率,B: 61.12%", document.content)
def test_upload_override_wins_over_the_env_default(self):
content = _workbook_with_percentages()
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
off = ExcelParser(xlsx_percent_as_text="false").parse_into_text(content)
with patch.dict(os.environ, {PERCENT_ENV: "false"}):
on = ExcelParser(xlsx_percent_as_text="true").parse_into_text(content)
self.assertNotIn("61.12%", off.content)
self.assertIn("B: 0.61123366", off.content)
self.assertIn("61.12%", on.content)
def test_absent_override_keeps_the_env_default(self):
content = _workbook_with_percentages()
with patch.dict(os.environ, {PERCENT_ENV: "true"}):
document = ExcelParser().parse_into_text(content)
self.assertIn("A: 毛利率,B: 61.12%", document.content)
if __name__ == "__main__":
unittest.main()

View File

@ -202,10 +202,16 @@ sequenceDiagram
三个辅助模块解决现实中的脏文件:
- **`excel_convert.py`**:魔数/`inspect_excel_format` 检测真实格式(xlsx/xls/xlsb/ods),为每种格式选 pandas engine(`xlrd`/`openpyxl`/`odf`);无法识别时(如 WPS `.et`、被改名的 csv)用 LibreOffice `convert-to xlsx` 归一化(`normalize_excel_bytes` 依次尝试 `.xlsx/.xls/.et/.csv` 后缀)。
- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。
- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。可选地**按显示格式物化百分比**(见下)。
- **`xlsx_repair.py`**:修复常见 XLSX 打包问题——`sharedStrings.xml` 路径大小写/位置不规范时重命名归位;manifest 引用了 sharedStrings 但包内缺失且工作表只用 inline string 时,从 `[Content_Types].xml` 与 workbook rels 中剥掉引用,使 openpyxl 能读。
XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。
**百分比格式**:Excel 的百分比是**显示格式而非存储值**——单元格实存 `0.61123366`,靠 `0.00%` 渲染成 `61.12%`,而 pandas/openpyxl 读的是原始值,格式被丢掉,于是 `61.12%` 变成 `0.6112`(**100 倍偏差**,财务报表类数据会直接把错数字喂给模型)。打开 `DOCREADER_XLSX_PERCENT_AS_TEXT`(或单次上传给 `parser_engine_overrides.xlsx_percent_as_text=true`,优先级高于环境变量)后,带百分比格式的数值单元格会被就地转成显示文本 `"61.12%"`(格式置 `General`),转换在**填合并单元格之前**完成——只有合并区域左上角带 `number_format`,被覆盖的格子是 `General`,先填合并会让第 2..n 行漏转。**只处理百分比**:货币/千分位只影响可读性(`1,234,567` 和 `1234567` 是同一个数),日期由 openpyxl 自动转 datetime。默认 **false**——开启后该单元格由数值变成字符串,会影响下游排序/计算,关闭时读取行为与升级前完全一致。
格式识别仅转换可可靠渲染的数字百分比模式(含颜色及正数/负数/零分段)。转义或引号中的 `%` 是字面量,不会触发乘 100;条件、填充和其他复杂自定义格式保留原始值,避免错误缩放。字面量百分号单元格也不会触发行提示推断。
同源文件偶尔有「一格忘了刷百分比格式」的单元格(如某财报 `D12` 折旧率 `0.0592` 格式为 `0.00_ ;[Red]\-0.00\ `)。把 `DOCREADER_XLSX_PCT_ROW_HINT_MIN` 设为 N(默认 **0=关闭**)后,同一行有 ≥N 个显式百分比格式单元格时,该行漏标的 `0<|v|<1` 数字也按百分比处理,每次命中打 INFO 日志可事后审计。
XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。百分比物化(若开启)发生在 `fill_merged_cells` 内部、填合并单元格之前。
#### ppt_convert.py / pptx_media.py(.ppt / .pptx,服务于 markitdown 引擎) {#_3-5-ppt-convert-py-pptx-media-py-ppt-pptx-服务于-markitdown-引擎}