From 44d90beea22a08b701fe3e33b650af9816a4f9f4 Mon Sep 17 00:00:00 2001 From: linus-liu-web <395141741@qq.com> Date: Sat, 17 Oct 2026 23:24:22 +0800 Subject: [PATCH] =?UTF-8?q?feat(docreader):=20=E4=B8=8A=E4=BC=A0=20HTML=20?= =?UTF-8?q?=E5=8F=AA=E7=B4=A2=E5=BC=95=E6=AD=A3=E6=96=87=EF=BC=8C=E4=B8=8E?= =?UTF-8?q?=20URL=20=E5=AF=BC=E5=85=A5=E8=B7=AF=E5=BE=84=E5=AF=B9=E9=BD=90?= =?UTF-8?q?=EF=BC=88=E5=BC=80=E5=85=B3=EF=BC=8C=E9=BB=98=E8=AE=A4=E5=85=B3?= =?UTF-8?q?=EF=BC=89=20(#4424)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 3 + docker-compose.yml | 3 + docreader/README.md | 4 + docreader/config.py | 9 ++ docreader/parser/html_parser.py | 112 +++++++++++++-- .../html_main_content/article_page.html | 76 ++++++++++ .../html_main_content/chrome_heavy_page.html | 90 ++++++++++++ .../testdata/html_main_content/form_page.html | 28 ++++ .../html_main_content/link_list_page.html | 27 ++++ .../html_main_content/short_notice.html | 19 +++ docreader/tests/test_config.py | 17 +++ docreader/tests/test_html_parser.py | 136 +++++++++++++++++- .../01-getting-started/04-configuration.md | 1 + .../03-features/03-document-parsing.md | 2 + 14 files changed, 516 insertions(+), 11 deletions(-) create mode 100644 docreader/testdata/html_main_content/article_page.html create mode 100644 docreader/testdata/html_main_content/chrome_heavy_page.html create mode 100644 docreader/testdata/html_main_content/form_page.html create mode 100644 docreader/testdata/html_main_content/link_list_page.html create mode 100644 docreader/testdata/html_main_content/short_notice.html diff --git a/.env.example b/.env.example index 207586b7..225fff97 100644 --- a/.env.example +++ b/.env.example @@ -425,6 +425,9 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434 # DOCREADER_DOCX_MAX_PAGES=0 # MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。 # DOCREADER_MARKITDOWN_MAX_WORKERS=1 +# 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。 +# 默认 false:整页转换,导航/侧栏/页脚也会进索引(与升级前一致)。 +# DOCREADER_HTML_MAIN_CONTENT=false # ----- XLSX / Excel ----- # 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112, # 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112 diff --git a/docker-compose.yml b/docker-compose.yml index 1d9c544c..9ee707b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -463,6 +463,9 @@ services: # ========== 解析通用 ========== - DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0} - DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1} + # 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。 + # 默认 false = 整页转换(导航/侧栏/页脚也会进索引),升级后行为不变。 + - DOCREADER_HTML_MAIN_CONTENT=${DOCREADER_HTML_MAIN_CONTENT:-false} # ========== XLSX / Excel ========== # 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存 # 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 diff --git a/docreader/README.md b/docreader/README.md index 9cd9546f..3121bae5 100644 --- a/docreader/README.md +++ b/docreader/README.md @@ -84,6 +84,10 @@ docreader: - `DOCREADER_PDF_RENDER_DPI`: 扫描 PDF 渲染 DPI(默认:200) - `DOCREADER_PDF_JPEG_QUALITY`: 扫描 PDF 输出 JPEG 质量(默认:85,范围会自动限制在 1-95) +### 上传 HTML 的正文抽取 + +- `DOCREADER_HTML_MAIN_CONTENT`: 上传的 `.html` / `.htm` 文件是否只索引正文(默认:`false`)。开启后用 trafilatura 抽取正文,与 URL 导入路径一致,导航栏 / 侧栏 / 页脚不再进入索引;抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时自动回退整页转换。默认关闭,升级后既有行为不变。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。 + ### OCR / VLM DocReader 自身不再内置 OCR 与 VLM 后端。扫描 PDF 会被渲染为 JPEG 图片后交由 Go App 侧调用 OCR/VLM 服务处理,相关配置请参考主项目文档。 diff --git a/docreader/config.py b/docreader/config.py index 1b9e5072..fcd71c58 100644 --- a/docreader/config.py +++ b/docreader/config.py @@ -61,6 +61,7 @@ class DocReaderConfig: odl_hybrid_mode: str odl_hybrid_fallback: bool odl_markdown_with_html: bool + html_main_content: bool pdf_render_max_workers: int pdf_render_parallelism: int pdf_render_dpi: int @@ -98,6 +99,12 @@ def load_config() -> DocReaderConfig: odl_markdown_with_html = _get_bool( ["DOCREADER_ODL_MARKDOWN_WITH_HTML"], False ) + # Index only the main content of an uploaded HTML file, the way the URL + # import path already does (trafilatura with a full-page fallback). It is + # off by default on purpose, so upgrading never changes what a knowledge + # base indexes for documents it already ingests: with it off every + # uploaded HTML file is converted in full, exactly as before. + html_main_content = _get_bool(["DOCREADER_HTML_MAIN_CONTENT"], False) pdf_render_max_workers = _get_int(["DOCREADER_PDF_RENDER_MAX_WORKERS"], 1) # Intra-document render parallelism: how many worker processes render the # scanned pages of a SINGLE PDF in parallel. pdfium is not thread-safe, so @@ -140,6 +147,7 @@ def load_config() -> DocReaderConfig: odl_hybrid_mode=odl_hybrid_mode, odl_hybrid_fallback=odl_hybrid_fallback, odl_markdown_with_html=odl_markdown_with_html, + html_main_content=html_main_content, pdf_render_max_workers=pdf_render_max_workers, pdf_render_parallelism=pdf_render_parallelism, pdf_render_dpi=pdf_render_dpi, @@ -168,6 +176,7 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]: "DOCREADER_ODL_HYBRID_MODE": cfg.odl_hybrid_mode, "DOCREADER_ODL_HYBRID_FALLBACK": cfg.odl_hybrid_fallback, "DOCREADER_ODL_MARKDOWN_WITH_HTML": cfg.odl_markdown_with_html, + "DOCREADER_HTML_MAIN_CONTENT": cfg.html_main_content, "DOCREADER_PDF_RENDER_MAX_WORKERS": cfg.pdf_render_max_workers, "DOCREADER_PDF_RENDER_PARALLELISM": cfg.pdf_render_parallelism, "DOCREADER_PDF_RENDER_DPI": cfg.pdf_render_dpi, diff --git a/docreader/parser/html_parser.py b/docreader/parser/html_parser.py index 79c69143..d8fffa96 100644 --- a/docreader/parser/html_parser.py +++ b/docreader/parser/html_parser.py @@ -1,17 +1,78 @@ """Static HTML file parser.""" -from bs4 import BeautifulSoup +import logging +from collections import Counter +from bs4 import BeautifulSoup +from trafilatura import extract + +from docreader.config import CONFIG from docreader.models.document import Document from docreader.parser.base_parser import BaseParser from docreader.parser.chain_parser import PipelineParser from docreader.parser.markdown_parser import MarkdownParser from docreader.parser.mhtml_parser import MHTMLParser +logger = logging.getLogger(__name__) + +# An uploaded HTML file is converted in full by default, so navigation, +# sidebars and footers are indexed as if they were document body. Setting +# DOCREADER_HTML_MAIN_CONTENT=true indexes only the main content instead, the +# way the URL import path (web_parser.py) already does with trafilatura. It is +# off by default on purpose, so upgrading never changes what a knowledge base +# indexes for documents it already ingests. +# +# Main-content extraction can fail in two ways: the page has no clear article +# body (a form page, a bare link list) or trafilatura misjudges it. Both +# thresholds below keep the full conversion in that case, which is also the +# upstream behavior. +_MIN_EXTRACTED_CHARS = 250 +# trafilatura falls back to a plain-text rendering below its own +# MIN_EXTRACTED_SIZE (250); that rendering drops heading levels, tables and +# link targets, so a page that lands between the two thresholds would lose +# structure instead of just losing its chrome. Bracketing at trafilatura's +# own floor keeps the full conversion in charge there. +_MIN_KEEP_RATIO = 0.15 + + +def extract_main_content(html: str) -> str: + """Extract the main content of a page with trafilatura. + + Returns the extracted markdown, or an empty string when trafilatura finds + nothing usable or raises; callers then keep the full conversion. + """ + try: + extracted = extract( + html, + output_format="markdown", + include_tables=True, + include_links=True, + include_formatting=True, + # Keep references in Markdown; Go can only resolve/download body + # images if their URLs survive this extraction step. + include_images=True, + ) + except Exception as exc: # never fail a parse the full conversion can serve + logger.warning("HTML main-content extraction failed: %s", exc) + return "" + return (extracted or "").strip() + class HTMLToMarkdownParser(BaseParser): """Convert uploaded HTML bytes to Markdown without browser or network access.""" + def _full_markdown(self, html: str) -> str: + """Convert the whole page, the way this parser always did.""" + return MHTMLParser( + file_name=self.file_name, + file_type=self.file_type, + extract_images=False, + ).html_to_markdown( + html, + strip_internal_links=False, + fallback_to_raw_html=False, + ) or "" + def parse_into_text(self, content: bytes) -> Document: if not content.strip(): return Document() @@ -19,16 +80,47 @@ class HTMLToMarkdownParser(BaseParser): # Inspect the original bytes so BOMs and HTML charset declarations are # honored before the shared Markdown conversion runs. html = BeautifulSoup(content, "lxml").decode() - markdown = MHTMLParser( - file_name=self.file_name, - file_type=self.file_type, - extract_images=False, - ).html_to_markdown( - html, - strip_internal_links=False, - fallback_to_raw_html=False, + + full = self._full_markdown(html) + if not CONFIG.html_main_content: + return Document(content=full) + + extracted = extract_main_content(html) + # Some short articles are repeated by trafilatura's fallback. Do not + # index extra copies or let them inflate the length/retention guards. + # Only reject repetitions absent from the full conversion: deliberate + # repeated paragraphs in the source must remain valid. + blocks = Counter(line.strip() for line in extracted.splitlines()) + for block, occurrences in blocks.items(): + if occurrences > 1 and len(block) >= 80 and 0 < full.count(block) < occurrences: + logger.info( + "HTML main-content extraction duplicated a body block; keeping the full conversion" + ) + return Document(content=full) + if len(extracted) < _MIN_EXTRACTED_CHARS: + logger.info( + "HTML main-content extraction returned %d chars (<%d); " + "keeping the full conversion", + len(extracted), + _MIN_EXTRACTED_CHARS, + ) + return Document(content=full) + if full and len(extracted) < len(full) * _MIN_KEEP_RATIO: + logger.info( + "HTML main-content extraction kept only %d chars of %d; " + "keeping the full conversion", + len(extracted), + len(full), + ) + return Document(content=full) + + logger.info( + "HTML main-content extraction applied: kept %d chars, " + "full conversion would be %d", + len(extracted), + len(full), ) - return Document(content=markdown or "") + return Document(content=extracted) class HTMLParser(PipelineParser): diff --git a/docreader/testdata/html_main_content/article_page.html b/docreader/testdata/html_main_content/article_page.html new file mode 100644 index 00000000..c6117093 --- /dev/null +++ b/docreader/testdata/html_main_content/article_page.html @@ -0,0 +1,76 @@ + + + + + 深入理解 RAG 的召回策略 - 示例技术博客 + + + + + + +
+
+
+

深入理解 RAG 的召回策略

+

检索增强生成的效果上限,几乎完全由召回阶段决定。如果召回的候选里没有包含答案所在的段落,后面再强的生成模型也无能为力。因此调优 RAG 的第一步,永远是先把召回率做上去。

+

关键词检索依赖倒排索引,对术语、编号、专有名词特别敏感;向量检索依赖语义空间,对同义改写和自然语言提问更友好,但对精确匹配反而容易失手。两者失败的样本分布并不重合。

+

混合检索把 BM25 的稀疏信号与向量召回融合在一起,常见做法是让两路各取前五十条候选,再用加权求和或倒数排名融合合并成一个列表。融合权重需要按语料特点调,不能直接照搬别人的参数。

+

重排模型只对前 N 条候选打分,N 通常取二十到一百之间。它比召回模型慢得多,因此不能用来扩大候选集,只能用来把已经召回的候选重新排序。把重排放在召回之后,是成本与效果之间最常见的折中。

+

分块策略同样影响召回。块太大会把多个主题混在一起,向量的语义被稀释;块太小则一句话被切成两半,答案的上下文不完整。实践中常用的做法是按标题切分,再对超长段落做二次切分,并保留一定的重叠。

+

评估召回质量不要只看最终答案对不对,而要单独测量命中率与排序位置。把每次线上提问的召回结果落盘,定期人工标注一批,是发现召回退化最直接的手段。

+ + + + + + + + + +
策略优点代价
关键词检索术语与编号精确匹配同义改写召回不到
向量检索语义相近即可召回专有名词容易漂移
混合检索两路互补,召回率最高需要调融合权重
+

延伸阅读:分块器的实现细节。

+
+
+ + +
+ + + + diff --git a/docreader/testdata/html_main_content/chrome_heavy_page.html b/docreader/testdata/html_main_content/chrome_heavy_page.html new file mode 100644 index 00000000..55d8e0aa --- /dev/null +++ b/docreader/testdata/html_main_content/chrome_heavy_page.html @@ -0,0 +1,90 @@ + + + + + 发布说明 - 示例开发者门户 + + +
+ +
+ +
+ + +
+
+

发布说明 2025-09

+

本次发布只修复了一个问题:批量导入任务在遇到空文件时不再中断整个批次。当上传的文件列表里混入 0 字节的条目时,任务会跳过该条目并继续处理剩余文件,同时在任务详情里给出跳过原因。升级后无需修改任何配置,历史任务的重试行为也保持一致;如需在升级前临时规避,可以先把空文件从列表里删除再重新提交。

+
+
+ + +
+ + + + diff --git a/docreader/testdata/html_main_content/form_page.html b/docreader/testdata/html_main_content/form_page.html new file mode 100644 index 00000000..a3116105 --- /dev/null +++ b/docreader/testdata/html_main_content/form_page.html @@ -0,0 +1,28 @@ + + + + + 账号登录 + + +
+ +
+
+

账号登录

+
+ + + + + + + + 忘记密码? +
+
+ + + diff --git a/docreader/testdata/html_main_content/link_list_page.html b/docreader/testdata/html_main_content/link_list_page.html new file mode 100644 index 00000000..eee117a0 --- /dev/null +++ b/docreader/testdata/html_main_content/link_list_page.html @@ -0,0 +1,27 @@ + + + + + 文档索引 + + +
+ +
+
+

文档索引

+ +
+ + + diff --git a/docreader/testdata/html_main_content/short_notice.html b/docreader/testdata/html_main_content/short_notice.html new file mode 100644 index 00000000..987ccb48 --- /dev/null +++ b/docreader/testdata/html_main_content/short_notice.html @@ -0,0 +1,19 @@ + + + + + 系统维护通知 + + +
+ +
+
+

系统维护通知

+

本服务将于 10 月 1 日 02:00 至 04:00 维护,期间暂停文档上传与检索。

+
+ + + diff --git a/docreader/tests/test_config.py b/docreader/tests/test_config.py index 7c1d6725..1c1ea77b 100644 --- a/docreader/tests/test_config.py +++ b/docreader/tests/test_config.py @@ -38,6 +38,23 @@ class DocReaderConfigTest(unittest.TestCase): self.assertIn("DOCREADER_PDF_RENDER_DPI", dumped) self.assertIn("DOCREADER_PDF_JPEG_QUALITY", dumped) + def test_html_main_content_is_off_by_default(self): + with patch.dict(os.environ, {}, clear=True): + cfg = config.load_config() + + self.assertFalse(cfg.html_main_content) + + def test_loads_html_main_content_env(self): + with patch.dict(os.environ, {"DOCREADER_HTML_MAIN_CONTENT": "true"}): + cfg = config.load_config() + + self.assertTrue(cfg.html_main_content) + + def test_dump_config_includes_html_main_content(self): + dumped = config.dump_config() + + self.assertIn("DOCREADER_HTML_MAIN_CONTENT", dumped) + if __name__ == "__main__": unittest.main() diff --git a/docreader/tests/test_html_parser.py b/docreader/tests/test_html_parser.py index 68edd901..3972fa72 100644 --- a/docreader/tests/test_html_parser.py +++ b/docreader/tests/test_html_parser.py @@ -1,10 +1,39 @@ import unittest +from dataclasses import replace from pathlib import Path +from unittest.mock import patch -from docreader.parser.html_parser import HTMLParser +import docreader.parser.html_parser as html_parser +from docreader.config import CONFIG +from docreader.parser.html_parser import ( + _MIN_EXTRACTED_CHARS, + _MIN_KEEP_RATIO, + HTMLParser, + extract_main_content, +) from docreader.parser.registry import BUILTIN_ENGINE, registry REPO_ROOT = Path(__file__).resolve().parents[2] +MAIN_CONTENT_FIXTURES = REPO_ROOT / "docreader" / "testdata" / "html_main_content" + + +def _parse_bytes( + content: bytes, *, main_content: bool, file_name: str = "page.html" +) -> str: + """Parse HTML through the upload path with the switch forced.""" + with patch.object( + html_parser, "CONFIG", replace(CONFIG, html_main_content=main_content) + ): + return HTMLParser(file_name=file_name, file_type="html").parse(content).content + + +def _parse_uploaded_html(name: str, *, main_content: bool) -> str: + """Parse a fixture through the upload path with the switch forced.""" + return _parse_bytes( + (MAIN_CONTENT_FIXTURES / name).read_bytes(), + main_content=main_content, + file_name=name, + ) class HTMLParserTest(unittest.TestCase): @@ -74,5 +103,110 @@ class HTMLParserTest(unittest.TestCase): self.assertIn("htm", builtin_types) +class HTMLMainContentTest(unittest.TestCase): + """DOCREADER_HTML_MAIN_CONTENT: opt-in main-content extraction on upload.""" + + def test_upload_indexes_the_whole_page_by_default(self): + content = _parse_uploaded_html("article_page.html", main_content=False) + + # Header nav, sidebar and footer all reach the index, as they always did. + self.assertIn("产品定价", content) + self.assertIn("订阅我们的新闻邮件", content) + self.assertIn("京ICP备00000000号-1", content) + self.assertIn("混合检索", content) + + def test_enabled_extraction_drops_page_chrome_and_keeps_the_article(self): + full = _parse_uploaded_html("article_page.html", main_content=False) + content = _parse_uploaded_html("article_page.html", main_content=True) + + self.assertNotIn("产品定价", content) # header nav + self.assertNotIn("订阅我们的新闻邮件", content) # sidebar + self.assertNotIn("京ICP备00000000号-1", content) # footer + # The article body survives in full, table and links included. + self.assertIn("# 深入理解 RAG 的召回策略", content) + self.assertIn("混合检索把 BM25 的稀疏信号与向量召回融合在一起", content) + self.assertIn("重排模型只对前 N 条候选打分", content) + self.assertIn("| 策略 | 优点 | 代价 |", content) + self.assertIn("两路互补,召回率最高", content) + self.assertIn("[分块器的实现细节](/blog/99)", content) + self.assertLess(len(content), len(full)) + + def test_enabled_extraction_preserves_body_image_references(self): + html = (MAIN_CONTENT_FIXTURES / "article_page.html").read_text(encoding="utf-8") + html = html.replace( + "", + '
Recall comparison
' + '
Local comparison
' + '', + ) + self.assertIn("https://example.com/chart.png", html) + content = _parse_bytes(html.encode(), main_content=True) + self.assertNotIn("产品定价", content) + self.assertIn("https://example.com/chart.png", content) + self.assertIn("images/local-chart.png", content) + self.assertIn("混合检索", content) + + def test_short_article_does_not_index_duplicate_body_text(self): + body = "正文说明知识库如何解析上传文件,以及如何把检索到的内容交给模型生成答案。" * 9 + html = f"

知识库解析说明

{body}

".encode() + content = _parse_bytes(html, main_content=True) + self.assertEqual(content.count(body), 1) + + def test_extractor_introduced_repetition_falls_back_but_source_repetition_does_not(self): + body = "A long paragraph about retrieval, citations and document parsing. " * 5 + html = f"

{body}

".encode() + full = _parse_bytes(html, main_content=False) + repeated = body.strip() + "\n\n" + body.strip() + with patch.object(html_parser, "extract_main_content", return_value=repeated): + self.assertEqual(_parse_bytes(html, main_content=True), full) + original_repeated = f"

{body}

{body}

".encode() + content = _parse_bytes(original_repeated, main_content=True) + self.assertEqual(content.count(body.strip()), 2) + + def test_falls_back_to_full_conversion_without_an_article_body(self): + for name, expected in ( + ("form_page.html", "忘记密码?"), + ("link_list_page.html", "第六章 常见问题"), + ): + with self.subTest(fixture=name): + full = _parse_uploaded_html(name, main_content=False) + content = _parse_uploaded_html(name, main_content=True) + + self.assertEqual(content, full) + self.assertIn(expected, content) + + def test_falls_back_when_extraction_keeps_too_little_of_a_long_page(self): + full = _parse_uploaded_html("chrome_heavy_page.html", main_content=False) + content = _parse_uploaded_html("chrome_heavy_page.html", main_content=True) + + # The length floor is passed here: what keeps the full conversion is the + # ratio guard against "trafilatura kept only a fragment of a long page". + html = (MAIN_CONTENT_FIXTURES / "chrome_heavy_page.html").read_text( + encoding="utf-8" + ) + extracted = extract_main_content(html) + self.assertGreaterEqual(len(extracted), _MIN_EXTRACTED_CHARS) + self.assertLess(len(extracted), len(full) * _MIN_KEEP_RATIO) + + self.assertEqual(content, full) + self.assertIn("批量导入任务在遇到空文件时不再中断整个批次", content) + self.assertIn("京ICP备00000000号-1", content) + + def test_short_page_is_not_reduced_to_nothing(self): + full = _parse_uploaded_html("short_notice.html", main_content=False) + content = _parse_uploaded_html("short_notice.html", main_content=True) + + self.assertEqual(content, full) + self.assertIn( + "本服务将于 10 月 1 日 02:00 至 04:00 维护,期间暂停文档上传与检索。", + content, + ) + + def test_empty_html_stays_empty(self): + content = _parse_bytes(b"", main_content=True, file_name="empty.html") + + self.assertEqual("", content) + + if __name__ == "__main__": unittest.main() diff --git a/website-docs/01-getting-started/04-configuration.md b/website-docs/01-getting-started/04-configuration.md index 3ba47373..8fa54828 100644 --- a/website-docs/01-getting-started/04-configuration.md +++ b/website-docs/01-getting-started/04-configuration.md @@ -279,6 +279,7 @@ AWS S3 的 `S3_ACCESS_KEY` / `S3_SECRET_KEY` 可以**同时留空**,此时走 | `DOCREADER_PDF_RENDER_DPI` / `DOCREADER_PDF_JPEG_QUALITY` / `DOCREADER_PDF_RENDER_MAX_EDGE` | 200 / 85 / 2000 | PDF 渲染 | | `DOCREADER_PDF_FORCE_SCANNED` / `DOCREADER_PDF_SCAN_IMAGE_RATIO` / `DOCREADER_PDF_SCAN_MIN_CHARS` | false / 代码默认 | 扫描件判定 | | `DOCREADER_ODL_HYBRID` / `DOCREADER_ODL_HYBRID_URL` / `DOCREADER_ODL_HYBRID_MODE` / `DOCREADER_ODL_HYBRID_FALLBACK` | off / http://odl-hybrid:5002 / auto / false | OpenDataLoader 混合解析 | +| `DOCREADER_HTML_MAIN_CONTENT` | false | 上传的 HTML 文件只索引正文(与 URL 导入同一抽取策略;抽取不足时回退整页转换) | | 其余 `DOCREADER_PDF_*`(词距/边栏/隐藏文本/嵌入图/图表区等 20+ 项) | 见 `docker-compose.yml` docreader 段注释 | PDF 版式与抽取精调 | | `DOCREADER_EXTERNAL_HTTP_PROXY` / `_HTTPS_PROXY` | 空 | docreader 出站抓取代理 | diff --git a/website-docs/03-features/03-document-parsing.md b/website-docs/03-features/03-document-parsing.md index 0cecf4b2..19f5bbea 100644 --- a/website-docs/03-features/03-document-parsing.md +++ b/website-docs/03-features/03-document-parsing.md @@ -254,6 +254,7 @@ PPT 系列**没有独立解析器**,由 `MarkitdownParser` 处理,这两个 - `HTMLToMarkdownParser` 先用 `BeautifulSoup(content, "lxml")` 解码原始字节——先看 BOM 与 HTML 内的 charset 声明,再交给统一的 Markdown 转换; - HTML → Markdown 复用 `MHTMLParser.html_to_markdown()`,但传入 `extract_images=False`(本地 HTML 文件没有 MIME 附件可抽)、`strip_internal_links=False`(保留站内链接)、`fallback_to_raw_html=False`(转换不出内容时返回空而不是塞一整块 ```` ```html ````); - 正文中通过 `` 引用的远程图片由 Go 侧补齐:`internal/infrastructure/docparser/image_resolver.go` 会带 SSRF 校验下载这些远程图片并转存到对象存储,再重写引用,使其与本地上传的图片走同一套 OCR / caption 流程。 +- 默认整页转换,导航栏 / 侧栏 / 页脚也会进入索引。设 `DOCREADER_HTML_MAIN_CONTENT=true` 后改用 trafilatura 只抽正文(与 URL 导入路径同一策略),抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时回退整页转换。该开关默认关闭,升级不会改变既有文档的索引内容。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。 #### xmind_parser.py — XMindParser(.xmind 思维导图) @@ -364,6 +365,7 @@ gRPC 响应中不再返回 chunks(`ReadResponse` 没有 chunk 字段);`Exc | `DOCREADER_ODL_HYBRID_MODE` | `auto` | hybrid 模式参数 | | `DOCREADER_ODL_HYBRID_FALLBACK` | false | hybrid 失败是否回退 | | `DOCREADER_ODL_MARKDOWN_WITH_HTML` | false | ODL markdown 允许 HTML | +| `DOCREADER_HTML_MAIN_CONTENT` | false | 上传 HTML 只索引正文(trafilatura 抽取,失败回退整页转换) | | `DOCREADER_PDF_RENDER_MAX_WORKERS` | 1 | PDF 渲染阶段限流(跨请求) | | `DOCREADER_PDF_RENDER_PARALLELISM` | `min(4, cpu)` | 单个 PDF 内扫描页渲染的 worker 进程数 | | `DOCREADER_PDF_RENDER_DPI` | 200 | 扫描页渲染 DPI |