diff --git a/.env.example b/.env.example index fb0529a9..207586b7 100644 --- a/.env.example +++ b/.env.example @@ -425,6 +425,17 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434 # DOCREADER_DOCX_MAX_PAGES=0 # MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。 # DOCREADER_MARKITDOWN_MAX_WORKERS=1 +# ----- XLSX / Excel ----- +# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112, +# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112 +# (100 倍偏差)。开启后百分比单元格变成字面量文本 "61.12%"。默认 false:开启会使单元格 +# 由数值变为字符串,改变下游排序/计算看到的内容;关闭时读取行为与升级前完全一致。 +# 单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖(true/false)。 +# DOCREADER_XLSX_PERCENT_AS_TEXT=false +# 兜底:同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比 +# 处理(源文件偶有漏刷格式的格子,每命中一次打 INFO 日志)。仅在上一开关开启时生效; +# 0=关闭该兜底(默认)。 +# DOCREADER_XLSX_PCT_ROW_HINT_MIN=0 # 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时用)。 # DOCREADER_EXTERNAL_HTTP_PROXY= # DOCREADER_EXTERNAL_HTTPS_PROXY= diff --git a/docker-compose.yml b/docker-compose.yml index d79b93d1..1d9c544c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -463,6 +463,16 @@ services: # ========== 解析通用 ========== - DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0} - DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1} + # ========== XLSX / Excel ========== + # 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存 + # 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 + # 61.12% 会变成 0.6112(100 倍偏差)。开启后该单元格变成字面量文本 "61.12%"。 + # 默认 false:开启会使单元格由数值变为字符串,改变下游排序/计算看到的内容, + # 存量部署行为保持不变。单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖。 + - DOCREADER_XLSX_PERCENT_AS_TEXT=${DOCREADER_XLSX_PERCENT_AS_TEXT:-false} + # 同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比 + # 处理(源文件偶有漏刷格式的格子)。仅在上一开关开启时生效;0=关闭(默认)。 + - DOCREADER_XLSX_PCT_ROW_HINT_MIN=${DOCREADER_XLSX_PCT_ROW_HINT_MIN:-0} # 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时) - DOCREADER_EXTERNAL_HTTP_PROXY=${DOCREADER_EXTERNAL_HTTP_PROXY:-} - DOCREADER_EXTERNAL_HTTPS_PROXY=${DOCREADER_EXTERNAL_HTTPS_PROXY:-} diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.py index 163ed6ed..645cf7f2 100644 --- a/docreader/parser/excel_parser.py +++ b/docreader/parser/excel_parser.py @@ -74,11 +74,16 @@ class ExcelParser(BaseParser): file_name: str = "", file_type: str | None = None, xlsx_first_row_as_header: Any = False, + xlsx_percent_as_text: Any = None, **kwargs: Any, ): super().__init__(file_name=file_name, file_type=file_type, **kwargs) self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header) - + # Tri-state: None (or an empty value) keeps the deployment default from + # DOCREADER_XLSX_PERCENT_AS_TEXT, an explicit value wins. Same + # precedence as the pdf_force_scanned parser override. + self.xlsx_percent_as_text = _parse_optional_bool(xlsx_percent_as_text) + def parse_into_text(self, content: bytes) -> Document: """Parse Excel file bytes into a Document object. @@ -100,7 +105,11 @@ class ExcelParser(BaseParser): source_blocks: List[dict] = [] start, end = 0, 0 - excel_file = _open_excel_file(content, file_type=self.file_type) + excel_file = _open_excel_file( + content, + file_type=self.file_type, + percent_as_text=self.xlsx_percent_as_text, + ) # Process each sheet in the Excel file for excel_sheet_name in excel_file.sheet_names: @@ -205,7 +214,19 @@ def _parse_bool(value: Any) -> bool: return str(value).strip().lower() in {"1", "true", "yes", "on"} -def _prepare_xlsx_bytes(data: bytes) -> bytes: +def _parse_optional_bool(value: Any) -> bool | None: + """Parse a tri-state parser override; None keeps the deployment default.""" + if value is None: + return None + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if not text: + return None + return text in {"1", "true", "yes", "y", "on"} + + +def _prepare_xlsx_bytes(data: bytes, percent_as_text: bool | None = None) -> bytes: repaired = repair_xlsx_bytes(data) if repaired is not None: data = repaired @@ -213,7 +234,7 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes: if readable is not None: data = readable try: - return fill_merged_cells_xlsx(data) + return fill_merged_cells_xlsx(data, percent_as_text=percent_as_text) except TypeError: # A non-conforming styles.xml (empty/malformed fills) makes openpyxl's # load_workbook raise before pandas ever sees the file (#3637). @@ -222,10 +243,14 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes: sanitized = sanitize_xlsx_styles(data) if sanitized is None: raise - return fill_merged_cells_xlsx(sanitized) + return fill_merged_cells_xlsx(sanitized, percent_as_text=percent_as_text) -def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile: +def _open_excel_file( + content: bytes, + file_type: str | None = None, + percent_as_text: bool | None = None, +) -> pd.ExcelFile: """Open an Excel workbook with explicit engine selection and fallbacks.""" data = content converted_via_soffice = False @@ -254,7 +279,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi engine = engine_for_format(ext) if ext == "xlsx": - data = _prepare_xlsx_bytes(data) + data = _prepare_xlsx_bytes(data, percent_as_text=percent_as_text) engine = "openpyxl" try: return pd.ExcelFile(BytesIO(data), engine=engine) @@ -269,7 +294,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi if repaired is None: raise logger.info("Repaired XLSX sharedStrings packaging before parse") - data = _prepare_xlsx_bytes(repaired) + data = _prepare_xlsx_bytes(repaired, percent_as_text=percent_as_text) continue except ValueError as exc: if converted_via_soffice or "cannot be determined" not in str(exc): diff --git a/docreader/parser/xlsx_merge.py b/docreader/parser/xlsx_merge.py index a0613bdb..12ccddeb 100644 --- a/docreader/parser/xlsx_merge.py +++ b/docreader/parser/xlsx_merge.py @@ -3,18 +3,185 @@ from __future__ import annotations import logging +import os +import re import zipfile from io import BytesIO logger = logging.getLogger(__name__) +# --- Percentage number formats (opt-in) -------------------------------------- +# Excel keeps a percentage as the stored fraction plus a display format: the +# cell holds 0.61123366 and "0.00%" renders it as 61.12%. openpyxl and pandas +# read only the fraction, so a value the user sees as 61.12% reaches the +# pipeline as 0.6112 — a 100x error that a model will copy verbatim into a +# report. When this switch is on, percentage cells are materialized as the +# literal text the spreadsheet shows ("61.12%") instead. +# +# It is off by default on purpose: the cell changes from a number into a +# string, so anything that sorts, aggregates or computes with it sees different +# input. With it off, every workbook is read exactly as before. A single upload +# can override it through parser_engine_overrides.xlsx_percent_as_text. +PERCENT_AS_TEXT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT" -def fill_merged_cells_xlsx(content: bytes) -> bytes: +# Only percentages are materialized. Currency and thousands separators change +# how a number is displayed, not what it means ("1,234,567" and 1234567 are the +# same number), so rewriting them would churn cells for no semantic gain, and +# date formats are already converted to datetimes by openpyxl. + +# --- Same-row hint fallback (opt-in, off by default) ------------------------- +# Workbooks occasionally leave one cell of a percentage row without a +# percentage format (a real financial statement had 折旧率 0.0592 formatted as +# "0.00_ ;[Red]\-0.00\ "). When a row carries at least this many explicitly +# percentage-formatted cells, an unformatted 0 < |value| < 1 number in that row +# is treated as a percentage too. Only consulted while percent materialization +# is on; 0 disables the fallback. +ROW_HINT_MIN_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN" +_DEFAULT_ROW_HINT_MIN = 0 + +# Only materialize a known numeric percentage pattern. Excel literals, conditions, +# scaling and arbitrary decorations require a full format renderer; leave those +# values untouched rather than guessing that every '%' means multiply by 100. +_PERCENT_PATTERN = re.compile( + r"(?P-?)(?P\()?(?P#,##)?0" + r"(?:\.(?P0+))?%(?P\))?" +) +_FORMAT_COLOR = re.compile( + r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color[0-9]+)\]", re.I +) + + +def _env_bool(name: str, default: bool) -> bool: + value = os.environ.get(name) + if value is None or not str(value).strip(): + return default + return str(value).strip().lower() in {"1", "true", "yes", "y", "on"} + + +def _env_int(name: str, default: int) -> int: + try: + return int(str(os.environ.get(name, default)).strip()) + except (TypeError, ValueError): + return default + + +def percent_as_text_default() -> bool: + """Deployment-wide default for materializing percentage formats. + + Read per call (not at import time) so the value can be flipped without a + module reload; a per-upload override still wins over it. + """ + return _env_bool(PERCENT_AS_TEXT_ENV, False) + + +def _format_percentage(value, number_format: str): + """Render supported Excel percentage formats, leaving other formats intact.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not number_format or "%" not in number_format: + return None + # Quoted/escaped percentages are literal text, not percentage operators. + # Reject unsupported conditions across all sections before selecting one. + cleaned = _FORMAT_COLOR.sub("", number_format) + if any(char in cleaned for char in ('"', "\\", "_", "*", "[", "]")): + return None + sections = cleaned.split(";") + if len(sections) > 4: + return None + section_index = 1 if value < 0 and len(sections) >= 2 else 0 + if value == 0 and len(sections) >= 3: + section_index = 2 + match = _PERCENT_PATTERN.fullmatch(sections[section_index].strip()) + if not match or bool(match["open"]) != bool(match["close"]): + return None + decimals = len(match["decimals"] or "") + # Excel's negative section formats the magnitude; a single section instead + # supplies the minus sign automatically. + amount = abs(value) if section_index == 1 else value + grouping = "," if match["group"] else "" + try: + number = f"{float(amount) * 100:{grouping}.{decimals}f}" + return f"{match['sign']}{match['open'] or ''}{number}%{match['close'] or ''}" + except (TypeError, ValueError, OverflowError): + return None + + +def _materialize_percentage_cells(ws, row_hint_min: int) -> tuple[int, int]: + """Rewrite the percentage-formatted numbers of *ws* as their display text. + + Must run before merged ranges are filled: only the master cell of a merge + carries a number_format while the covered cells are General, so filling + first would copy a bare fraction into rows 2..n and leave it unconverted — + the same row would then show both "0.55%" and 0.005509433890047721. + + Returns ``(materialized, hinted)`` cell counts. + """ + row_hints: dict = {} + if row_hint_min > 0: + # Count the explicit percentage formats before converting anything: + # the conversion below resets number_format to General. + for row in ws.iter_rows(): + count = sum( + 1 + for cell in row + if type(cell.value) in (int, float) # noqa: E721 - excludes bool + and _format_percentage(cell.value, cell.number_format) is not None + ) + if count: + row_hints[row[0].row] = count + + materialized = 0 + hinted = 0 + for row in ws.iter_rows(): + hint = row_hints.get(row[0].row, 0) + for cell in row: + if type(cell.value) not in (int, float): # noqa: E721 - excludes bool + continue + text = _format_percentage(cell.value, cell.number_format) + if ( + text is None + and "%" not in str(cell.number_format or "") + and hint >= row_hint_min > 0 + and 0 < abs(cell.value) < 1 + ): + # Fallback for a cell that lost its percentage format while its + # row kept one. Logged because it rewrites an unformatted value. + text = f"{float(cell.value) * 100:.2f}%" + hinted += 1 + logger.info( + "XLSX pre-parse: %s!%s%d has no percentage format but its row " + "does; reading it as %.2f%% (stored value %r)", + ws.title, + cell.column_letter, + cell.row, + float(cell.value) * 100, + cell.value, + ) + if text is None: + continue + cell.value = text + cell.number_format = "General" + materialized += 1 + return materialized, hinted + + +def fill_merged_cells_xlsx( + content: bytes, percent_as_text: bool | None = None +) -> bytes: """Unmerge ranges and copy the master cell value into every covered cell. openpyxl only stores values on the top-left cell of a merge; pandas then sees NaN in the rest. Filling makes row-wise RAG chunks retain context. + + ``percent_as_text`` also materializes percentage-formatted numbers as the + text the spreadsheet displays ("0.6112" -> "61.12%"), because the stored + fraction is 100x off from what a reader sees. ``None`` (the default) uses + the ``DOCREADER_XLSX_PERCENT_AS_TEXT`` deployment default, which is off, so + the default path is unchanged. """ + if percent_as_text is None: + percent_as_text = percent_as_text_default() + if not zipfile.is_zipfile(BytesIO(content)): return content @@ -22,7 +189,21 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes: wb = load_workbook(BytesIO(content), data_only=True) changed = False + materialized_cells = 0 + hinted_cells = 0 + row_hint_min = ( + _env_int(ROW_HINT_MIN_ENV, _DEFAULT_ROW_HINT_MIN) if percent_as_text else 0 + ) + for ws in wb.worksheets: + if percent_as_text: + # Before the merge fill below — see _materialize_percentage_cells. + materialized, hinted = _materialize_percentage_cells(ws, row_hint_min) + materialized_cells += materialized + hinted_cells += hinted + if materialized: + changed = True + if not ws.merged_cells.ranges: continue for merge_range in list(ws.merged_cells.ranges): @@ -52,5 +233,13 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes: out = BytesIO() wb.save(out) + if materialized_cells: + logger.info( + "Materialized %d percentage cell(s) in XLSX before parse " + "(%d recovered by the same-row hint fallback, threshold=%d)", + materialized_cells, + hinted_cells, + row_hint_min, + ) logger.info("Filled merged cells in XLSX before parse") return out.getvalue() diff --git a/docreader/tests/test_xlsx_percent_format.py b/docreader/tests/test_xlsx_percent_format.py new file mode 100644 index 00000000..5a8b81de --- /dev/null +++ b/docreader/tests/test_xlsx_percent_format.py @@ -0,0 +1,377 @@ +"""Tests for materializing XLSX percentage number formats (opt-in). + +Excel keeps a percentage as the underlying fraction plus a display format: +the cell stores ``0.61123366`` and ``0.00%`` renders it as ``61.12%``. +openpyxl and pandas read the fraction and drop the format, so a cell the user +sees as ``61.12%`` reaches the pipeline as ``0.6112`` — a 100x error that a +model will happily copy into a report. + +When ``DOCREADER_XLSX_PERCENT_AS_TEXT`` is on (or the per-upload +``parser_engine_overrides`` key ``xlsx_percent_as_text`` is set), percentage +cells are materialized as the literal text the user sees. The switch is off by +default because the value changes from a number into a string, so anything that +sorts or aggregates the cell would see different input; with it off every +workbook is read exactly as before. +""" + +import datetime +import io +import os +import unittest +from unittest.mock import patch + +import openpyxl + +from docreader.parser.excel_parser import ExcelParser +from docreader.parser.parser import Parser +from docreader.parser.xlsx_merge import fill_merged_cells_xlsx, _format_percentage + +PERCENT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT" +ROW_HINT_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN" +OVERRIDE_KEY = "xlsx_percent_as_text" + + +def _workbook_with_percentages() -> bytes: + """A real .xlsx mixing percentages, a merge, plain numbers and dates. + + Layout (1-based; ``B3:B4`` is merged, so row 4's B cell is covered and has + no format of its own — exactly like the yearly columns of a financial + statement, where a label merge covers the row below): + + A B C + 1 科目 2024 2025 + 2 毛利率 0.61123366 (0.00%) 0.5512345 (0.00%) + 3 收入占比 0.0833 (0.00%) 0.0912 (0.00%) + 4 收入占比(续) 0.005509433890047721 (0.00%) + 5 营业收入 1234567.89 (#,##0.00) 9876543.21 (#,##0.00) + 6 报告日 2025-01-31 (yyyy-mm-dd) 2024-12-31 (yyyy-mm-dd) + 7 汇总 0.5 (General) 6100 (General) + + Row 4 is the regression case: it carries its own percentage cell in C, so a + merge fill that runs before the percentage pass leaves that row showing both + ``0.55%`` and a raw ``0.0833``. + """ + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "财务" + ws.append(["科目", "2024", "2025"]) + ws.append(["毛利率", 0.61123366, 0.5512345]) + ws.append(["收入占比", 0.0833, 0.0912]) + ws.append(["收入占比(续)", None, 0.005509433890047721]) + ws.append(["营业收入", 1234567.89, 9876543.21]) + ws.append(["报告日", datetime.date(2025, 1, 31), datetime.date(2024, 12, 31)]) + ws.append(["汇总", 0.5, 6100]) + + for row in (2, 3): + ws.cell(row=row, column=2).number_format = "0.00%" + ws.cell(row=row, column=3).number_format = "0.00%" + ws["C4"].number_format = "0.00%" + # Merged after the formats are set: openpyxl keeps only the master (B3) + # style, which is what makes the covered cell's format General. + ws.merge_cells("B3:B4") + for row in (5,): + ws.cell(row=row, column=2).number_format = "#,##0.00" + ws.cell(row=row, column=3).number_format = "#,##0.00" + ws["B6"].number_format = "yyyy-mm-dd" + ws["C6"].number_format = "yyyy-mm-dd" + + bio = io.BytesIO() + wb.save(bio) + return bio.getvalue() + + +def _workbook_with_unformatted_cell_in_percentage_row() -> bytes: + """A percentage row (3 formatted cells) with one cell left General (D2).""" + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["项目", "2024", "2025", "2026", "2027"]) + ws.append(["折旧率", 0.0592, 0.0601, 0.5, 0.0612]) + for col in (2, 3, 5): + ws.cell(row=2, column=col).number_format = "0.00%" + bio = io.BytesIO() + wb.save(bio) + return bio.getvalue() + + +def _workbook_without_merges() -> bytes: + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["项目", "值"]) + ws.append(["占比", 0.25]) + ws["B2"].number_format = "0%" + bio = io.BytesIO() + wb.save(bio) + return bio.getvalue() + + +def _read_cells(content: bytes, *coordinates: str): + """Return the values of *coordinates* on the first worksheet.""" + wb = openpyxl.load_workbook(io.BytesIO(content), data_only=True) + ws = wb.worksheets[0] + return [ws[c].value for c in coordinates] + + +def _upstream_fill_merged_cells_xlsx(content: bytes) -> bytes: + """Frozen copy of the pre-change implementation, for byte-identity checks. + + Kept verbatim (docreader/parser/xlsx_merge.py before the percentage switch) + so the default path of the new function can be compared byte for byte with + the behavior existing deployments have today. + """ + import zipfile + from io import BytesIO + + if not zipfile.is_zipfile(BytesIO(content)): + return content + + from openpyxl import load_workbook + + wb = load_workbook(BytesIO(content), data_only=True) + changed = False + for ws in wb.worksheets: + if not ws.merged_cells.ranges: + continue + for merge_range in list(ws.merged_cells.ranges): + master_value = ws.cell(merge_range.min_row, merge_range.min_col).value + ws.unmerge_cells(str(merge_range)) + for row in range(merge_range.min_row, merge_range.max_row + 1): + for col in range(merge_range.min_col, merge_range.max_col + 1): + ws.cell(row, col).value = master_value + changed = True + + if not changed: + return content + + active_sheet = wb.active + for chart_sheet in list(wb.chartsheets): + wb.remove(chart_sheet) + if active_sheet in wb.worksheets: + wb.active = active_sheet + elif wb.worksheets: + wb.active = wb.worksheets[0] + for ws in wb.worksheets: + ws._charts.clear() + + out = BytesIO() + wb.save(out) + return out.getvalue() + + +class PercentMaterializationOffTests(unittest.TestCase): + """Default behavior: unchanged, and byte-identical to the old function.""" + + def test_default_keeps_the_raw_fraction(self): + content = _workbook_with_percentages() + + self.assertEqual( + fill_merged_cells_xlsx(content), _upstream_fill_merged_cells_xlsx(content) + ) + + values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "B4") + self.assertEqual(values, [0.61123366, 0.5512345, 0.0833, 0.0833]) + + def test_excel_parser_default_keeps_the_raw_fraction(self): + document = ExcelParser().parse_into_text(_workbook_with_percentages()) + + self.assertIn("A: 毛利率,B: 0.61123366", document.content) + self.assertNotIn("61.12%", document.content) + + def test_switch_off_is_byte_identical_to_the_upstream_implementation(self): + for build in ( + _workbook_with_percentages, + _workbook_without_merges, + _workbook_with_unformatted_cell_in_percentage_row, + ): + content = build() + with self.subTest(fixture=build.__name__): + # Explicitly off, so a stray env var in the calling shell + # cannot turn this into an accidental behavior test. + with patch.dict(os.environ, {PERCENT_ENV: "0"}): + self.assertEqual( + fill_merged_cells_xlsx(content), + _upstream_fill_merged_cells_xlsx(content), + ) + + def test_env_var_off_forms_keep_the_raw_fraction(self): + content = _workbook_with_percentages() + for raw in ("0", "false", "FALSE", "no", "off", ""): + with self.subTest(value=raw): + with patch.dict(os.environ, {PERCENT_ENV: raw}): + self.assertEqual( + _read_cells(fill_merged_cells_xlsx(content), "B2"), + [0.61123366], + ) + + +class PercentMaterializationOnTests(unittest.TestCase): + """Enabled: percentages become the text the spreadsheet displays.""" + + def test_percent_cells_become_display_text(self): + content = _workbook_with_percentages() + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + values = _read_cells(fill_merged_cells_xlsx(content), "B2", "C2", "B3", "C4") + + self.assertEqual(values, ["61.12%", "55.12%", "8.33%", "0.55%"]) + + def test_merged_rows_below_the_master_are_materialized_too(self): + """The order pitfall: B4 is covered by B3:B4 and has no format itself.""" + content = _workbook_with_percentages() + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + b3, b4 = _read_cells(fill_merged_cells_xlsx(content), "B3", "B4") + + self.assertEqual(b3, "8.33%") + self.assertEqual(b4, "8.33%") + + def test_no_row_shows_both_a_percentage_and_a_raw_fraction(self): + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + document = ExcelParser().parse_into_text(_workbook_with_percentages()) + + self.assertIn("A: 毛利率,B: 61.12%,C: 55.12%", document.content) + self.assertIn("A: 收入占比,B: 8.33%,C: 9.12%", document.content) + # Row 4 keeps its own percentage cell (C) and its merged B cell: both + # must be text, or the same row would read as two different scales. + self.assertIn("A: 收入占比(续),B: 8.33%,C: 0.55%", document.content) + for raw in ("0.61123366", "0.5512345", "0.0833", "0.0912", "0.005509433890047721"): + self.assertNotIn(raw, document.content) + + def test_currency_plain_numbers_and_dates_are_not_touched(self): + content = _workbook_with_percentages() + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + income, income_2025, plain, whole, start, end = _read_cells( + fill_merged_cells_xlsx(content), "B5", "C5", "B7", "C7", "B6", "C6" + ) + + # Thousands separators / currency only affect readability. + self.assertEqual(income, 1234567.89) + self.assertEqual(income_2025, 9876543.21) + # 0.5 has no percentage format, so it must stay a number. + self.assertEqual(plain, 0.5) + self.assertEqual(whole, 6100) + # openpyxl already turns date formats into datetimes. + self.assertEqual(start, datetime.datetime(2025, 1, 31)) + self.assertEqual(end, datetime.datetime(2024, 12, 31)) + + def test_decimals_follow_the_number_format(self): + """The text mirrors what the format displays: 0%, 0.0%, 0.00%.""" + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["项目", "0%", "0.0%", "0.00%", "red"]) + ws.append(["毛利率", 0.61123366, 0.61123366, 0.61123366, 0.61123366]) + for col, fmt in ( + (2, "0%"), + (3, "0.0%"), + (4, "#,##0.00%"), + (5, "0.00%;[Red]-0.00%"), + ): + ws.cell(row=2, column=col).number_format = fmt + bio = io.BytesIO() + wb.save(bio) + + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + values = _read_cells( + fill_merged_cells_xlsx(bio.getvalue()), "B2", "C2", "D2", "E2" + ) + + self.assertEqual(values, ["61%", "61.1%", "61.12%", "61.12%"]) + + def test_unformatted_cell_in_a_percentage_row_needs_the_opt_in_fallback(self): + content = _workbook_with_unformatted_cell_in_percentage_row() + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + self.assertEqual( + _read_cells(fill_merged_cells_xlsx(content), "D2"), [0.5] + ) + with patch.dict( + os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "3"} + ): + self.assertEqual( + _read_cells(fill_merged_cells_xlsx(content), "D2"), ["50.00%"] + ) + + +class PercentageFormatSyntaxTests(unittest.TestCase): + def test_literal_and_conditional_percent_formats_are_not_scaled(self): + for fmt in (r"0.00\%", '0.00"%"', '[>1]0%;0.00%', '0.00%" growth"'): + with self.subTest(fmt=fmt): + self.assertIsNone(_format_percentage(0.5, fmt)) + + def test_selects_the_section_for_the_value(self): + for value, fmt, expected in ( + (0.5, "0%;0.00%", "50%"), + (-0.5, "0%;[Red]-0.00%", "-50.00%"), + (-0.5, "0%;(0.00%)", "(50.00%)"), + (0, "0%;0.00%;0.0%", "0.0%"), + (-0.5, "0.00%", "-50.00%"), + (12.345, "#,##0.00%", "1,234.50%"), + ): + with self.subTest(value=value, fmt=fmt): + self.assertEqual(_format_percentage(value, fmt), expected) + + def test_row_hint_does_not_convert_literal_percent_or_count_it_as_a_hint(self): + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["label", "percent", "literal", "plain"]) + ws.append(["mixed", 0.5, 0.5, 0.5]) + ws.append(["literal only", 0.5, 0.5, 0.5]) + ws["B2"].number_format = "0%" + ws["C2"].number_format = r"0.00\%" + ws["B3"].number_format = '0.00"%"' + data = io.BytesIO() + wb.save(data) + with patch.dict(os.environ, {PERCENT_ENV: "true", ROW_HINT_ENV: "1"}): + result = fill_merged_cells_xlsx(data.getvalue()) + self.assertEqual(_read_cells(result, "B2", "C2", "D2"), ["50%", 0.5, "50.00%"]) + self.assertEqual(_read_cells(result, "B3", "C3", "D3"), [0.5, 0.5, 0.5]) + + +class PercentMaterializationSwitchTests(unittest.TestCase): + """The switch itself: env default, per-upload override, precedence.""" + + def test_env_var_true_forms_enable_materialization(self): + content = _workbook_with_percentages() + for raw in ("1", "true", "TRUE", "yes", "y", "on"): + with self.subTest(value=raw): + with patch.dict(os.environ, {PERCENT_ENV: raw}): + self.assertEqual( + _read_cells(fill_merged_cells_xlsx(content), "B2"), + ["61.12%"], + ) + + def test_excel_parser_accepts_the_override_key(self): + document = ExcelParser(xlsx_percent_as_text="true").parse_into_text( + _workbook_with_percentages() + ) + + self.assertIn("A: 毛利率,B: 61.12%", document.content) + + def test_parser_engine_overrides_key_enables_materialization(self): + document = Parser().parse_file( + "report.xlsx", + "xlsx", + _workbook_with_percentages(), + engine_overrides={OVERRIDE_KEY: "true"}, + ) + + self.assertIn("A: 毛利率,B: 61.12%", document.content) + + def test_upload_override_wins_over_the_env_default(self): + content = _workbook_with_percentages() + + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + off = ExcelParser(xlsx_percent_as_text="false").parse_into_text(content) + with patch.dict(os.environ, {PERCENT_ENV: "false"}): + on = ExcelParser(xlsx_percent_as_text="true").parse_into_text(content) + + self.assertNotIn("61.12%", off.content) + self.assertIn("B: 0.61123366", off.content) + self.assertIn("61.12%", on.content) + + def test_absent_override_keeps_the_env_default(self): + content = _workbook_with_percentages() + with patch.dict(os.environ, {PERCENT_ENV: "true"}): + document = ExcelParser().parse_into_text(content) + + self.assertIn("A: 毛利率,B: 61.12%", document.content) + + +if __name__ == "__main__": + unittest.main() diff --git a/website-docs/03-features/03-document-parsing.md b/website-docs/03-features/03-document-parsing.md index e59fd781..0cecf4b2 100644 --- a/website-docs/03-features/03-document-parsing.md +++ b/website-docs/03-features/03-document-parsing.md @@ -202,10 +202,16 @@ sequenceDiagram 三个辅助模块解决现实中的脏文件: - **`excel_convert.py`**:魔数/`inspect_excel_format` 检测真实格式(xlsx/xls/xlsb/ods),为每种格式选 pandas engine(`xlrd`/`openpyxl`/`odf`);无法识别时(如 WPS `.et`、被改名的 csv)用 LibreOffice `convert-to xlsx` 归一化(`normalize_excel_bytes` 依次尝试 `.xlsx/.xls/.et/.csv` 后缀)。 -- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。 +- **`xlsx_merge.py`**:`fill_merged_cells_xlsx` 解除合并单元格并把左上角主值**复制到覆盖区域的每个单元格**——openpyxl 只在左上角存值,pandas 会把其余读成 NaN,填充后按行分块的 RAG chunk 才能保留上下文。可选地**按显示格式物化百分比**(见下)。 - **`xlsx_repair.py`**:修复常见 XLSX 打包问题——`sharedStrings.xml` 路径大小写/位置不规范时重命名归位;manifest 引用了 sharedStrings 但包内缺失且工作表只用 inline string 时,从 `[Content_Types].xml` 与 workbook rels 中剥掉引用,使 openpyxl 能读。 -XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。 +**百分比格式**:Excel 的百分比是**显示格式而非存储值**——单元格实存 `0.61123366`,靠 `0.00%` 渲染成 `61.12%`,而 pandas/openpyxl 读的是原始值,格式被丢掉,于是 `61.12%` 变成 `0.6112`(**100 倍偏差**,财务报表类数据会直接把错数字喂给模型)。打开 `DOCREADER_XLSX_PERCENT_AS_TEXT`(或单次上传给 `parser_engine_overrides.xlsx_percent_as_text=true`,优先级高于环境变量)后,带百分比格式的数值单元格会被就地转成显示文本 `"61.12%"`(格式置 `General`),转换在**填合并单元格之前**完成——只有合并区域左上角带 `number_format`,被覆盖的格子是 `General`,先填合并会让第 2..n 行漏转。**只处理百分比**:货币/千分位只影响可读性(`1,234,567` 和 `1234567` 是同一个数),日期由 openpyxl 自动转 datetime。默认 **false**——开启后该单元格由数值变成字符串,会影响下游排序/计算,关闭时读取行为与升级前完全一致。 + +格式识别仅转换可可靠渲染的数字百分比模式(含颜色及正数/负数/零分段)。转义或引号中的 `%` 是字面量,不会触发乘 100;条件、填充和其他复杂自定义格式保留原始值,避免错误缩放。字面量百分号单元格也不会触发行提示推断。 + +同源文件偶尔有「一格忘了刷百分比格式」的单元格(如某财报 `D12` 折旧率 `0.0592` 格式为 `0.00_ ;[Red]\-0.00\ `)。把 `DOCREADER_XLSX_PCT_ROW_HINT_MIN` 设为 N(默认 **0=关闭**)后,同一行有 ≥N 个显式百分比格式单元格时,该行漏标的 `0<|v|<1` 数字也按百分比处理,每次命中打 INFO 日志可事后审计。 + +XLSX 读取前统一走 `repair → fill_merged_cells` 预处理,并用 `header=None` + A/B/C 列字母作为稳定列名(xls 则先尝试首行做表头,遇 `Unnamed:` 列回退列字母)。百分比物化(若开启)发生在 `fill_merged_cells` 内部、填合并单元格之前。 #### ppt_convert.py / pptx_media.py(.ppt / .pptx,服务于 markitdown 引擎) {#_3-5-ppt-convert-py-pptx-media-py-ppt-pptx-服务于-markitdown-引擎}