From 44d90beea22a08b701fe3e33b650af9816a4f9f4 Mon Sep 17 00:00:00 2001
From: linus-liu-web <395141741@qq.com>
Date: Sat, 17 Oct 2026 23:24:22 +0800
Subject: [PATCH] =?UTF-8?q?feat(docreader):=20=E4=B8=8A=E4=BC=A0=20HTML=20?=
=?UTF-8?q?=E5=8F=AA=E7=B4=A2=E5=BC=95=E6=AD=A3=E6=96=87=EF=BC=8C=E4=B8=8E?=
=?UTF-8?q?=20URL=20=E5=AF=BC=E5=85=A5=E8=B7=AF=E5=BE=84=E5=AF=B9=E9=BD=90?=
=?UTF-8?q?=EF=BC=88=E5=BC=80=E5=85=B3=EF=BC=8C=E9=BB=98=E8=AE=A4=E5=85=B3?=
=?UTF-8?q?=EF=BC=89=20(#4424)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.env.example | 3 +
docker-compose.yml | 3 +
docreader/README.md | 4 +
docreader/config.py | 9 ++
docreader/parser/html_parser.py | 112 +++++++++++++--
.../html_main_content/article_page.html | 76 ++++++++++
.../html_main_content/chrome_heavy_page.html | 90 ++++++++++++
.../testdata/html_main_content/form_page.html | 28 ++++
.../html_main_content/link_list_page.html | 27 ++++
.../html_main_content/short_notice.html | 19 +++
docreader/tests/test_config.py | 17 +++
docreader/tests/test_html_parser.py | 136 +++++++++++++++++-
.../01-getting-started/04-configuration.md | 1 +
.../03-features/03-document-parsing.md | 2 +
14 files changed, 516 insertions(+), 11 deletions(-)
create mode 100644 docreader/testdata/html_main_content/article_page.html
create mode 100644 docreader/testdata/html_main_content/chrome_heavy_page.html
create mode 100644 docreader/testdata/html_main_content/form_page.html
create mode 100644 docreader/testdata/html_main_content/link_list_page.html
create mode 100644 docreader/testdata/html_main_content/short_notice.html
diff --git a/.env.example b/.env.example
index 207586b7..225fff97 100644
--- a/.env.example
+++ b/.env.example
@@ -425,6 +425,9 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# DOCREADER_DOCX_MAX_PAGES=0
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
+# 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。
+# 默认 false:整页转换,导航/侧栏/页脚也会进索引(与升级前一致)。
+# DOCREADER_HTML_MAIN_CONTENT=false
# ----- XLSX / Excel -----
# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112
diff --git a/docker-compose.yml b/docker-compose.yml
index 1d9c544c..9ee707b9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -463,6 +463,9 @@ services:
# ========== 解析通用 ==========
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
+ # 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。
+ # 默认 false = 整页转换(导航/侧栏/页脚也会进索引),升级后行为不变。
+ - DOCREADER_HTML_MAIN_CONTENT=${DOCREADER_HTML_MAIN_CONTENT:-false}
# ========== XLSX / Excel ==========
# 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
# 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的
diff --git a/docreader/README.md b/docreader/README.md
index 9cd9546f..3121bae5 100644
--- a/docreader/README.md
+++ b/docreader/README.md
@@ -84,6 +84,10 @@ docreader:
- `DOCREADER_PDF_RENDER_DPI`: 扫描 PDF 渲染 DPI(默认:200)
- `DOCREADER_PDF_JPEG_QUALITY`: 扫描 PDF 输出 JPEG 质量(默认:85,范围会自动限制在 1-95)
+### 上传 HTML 的正文抽取
+
+- `DOCREADER_HTML_MAIN_CONTENT`: 上传的 `.html` / `.htm` 文件是否只索引正文(默认:`false`)。开启后用 trafilatura 抽取正文,与 URL 导入路径一致,导航栏 / 侧栏 / 页脚不再进入索引;抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时自动回退整页转换。默认关闭,升级后既有行为不变。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。
+
### OCR / VLM
DocReader 自身不再内置 OCR 与 VLM 后端。扫描 PDF 会被渲染为 JPEG 图片后交由 Go App 侧调用 OCR/VLM 服务处理,相关配置请参考主项目文档。
diff --git a/docreader/config.py b/docreader/config.py
index 1b9e5072..fcd71c58 100644
--- a/docreader/config.py
+++ b/docreader/config.py
@@ -61,6 +61,7 @@ class DocReaderConfig:
odl_hybrid_mode: str
odl_hybrid_fallback: bool
odl_markdown_with_html: bool
+ html_main_content: bool
pdf_render_max_workers: int
pdf_render_parallelism: int
pdf_render_dpi: int
@@ -98,6 +99,12 @@ def load_config() -> DocReaderConfig:
odl_markdown_with_html = _get_bool(
["DOCREADER_ODL_MARKDOWN_WITH_HTML"], False
)
+ # Index only the main content of an uploaded HTML file, the way the URL
+ # import path already does (trafilatura with a full-page fallback). It is
+ # off by default on purpose, so upgrading never changes what a knowledge
+ # base indexes for documents it already ingests: with it off every
+ # uploaded HTML file is converted in full, exactly as before.
+ html_main_content = _get_bool(["DOCREADER_HTML_MAIN_CONTENT"], False)
pdf_render_max_workers = _get_int(["DOCREADER_PDF_RENDER_MAX_WORKERS"], 1)
# Intra-document render parallelism: how many worker processes render the
# scanned pages of a SINGLE PDF in parallel. pdfium is not thread-safe, so
@@ -140,6 +147,7 @@ def load_config() -> DocReaderConfig:
odl_hybrid_mode=odl_hybrid_mode,
odl_hybrid_fallback=odl_hybrid_fallback,
odl_markdown_with_html=odl_markdown_with_html,
+ html_main_content=html_main_content,
pdf_render_max_workers=pdf_render_max_workers,
pdf_render_parallelism=pdf_render_parallelism,
pdf_render_dpi=pdf_render_dpi,
@@ -168,6 +176,7 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
"DOCREADER_ODL_HYBRID_MODE": cfg.odl_hybrid_mode,
"DOCREADER_ODL_HYBRID_FALLBACK": cfg.odl_hybrid_fallback,
"DOCREADER_ODL_MARKDOWN_WITH_HTML": cfg.odl_markdown_with_html,
+ "DOCREADER_HTML_MAIN_CONTENT": cfg.html_main_content,
"DOCREADER_PDF_RENDER_MAX_WORKERS": cfg.pdf_render_max_workers,
"DOCREADER_PDF_RENDER_PARALLELISM": cfg.pdf_render_parallelism,
"DOCREADER_PDF_RENDER_DPI": cfg.pdf_render_dpi,
diff --git a/docreader/parser/html_parser.py b/docreader/parser/html_parser.py
index 79c69143..d8fffa96 100644
--- a/docreader/parser/html_parser.py
+++ b/docreader/parser/html_parser.py
@@ -1,17 +1,78 @@
"""Static HTML file parser."""
-from bs4 import BeautifulSoup
+import logging
+from collections import Counter
+from bs4 import BeautifulSoup
+from trafilatura import extract
+
+from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.mhtml_parser import MHTMLParser
+logger = logging.getLogger(__name__)
+
+# An uploaded HTML file is converted in full by default, so navigation,
+# sidebars and footers are indexed as if they were document body. Setting
+# DOCREADER_HTML_MAIN_CONTENT=true indexes only the main content instead, the
+# way the URL import path (web_parser.py) already does with trafilatura. It is
+# off by default on purpose, so upgrading never changes what a knowledge base
+# indexes for documents it already ingests.
+#
+# Main-content extraction can fail in two ways: the page has no clear article
+# body (a form page, a bare link list) or trafilatura misjudges it. Both
+# thresholds below keep the full conversion in that case, which is also the
+# upstream behavior.
+_MIN_EXTRACTED_CHARS = 250
+# trafilatura falls back to a plain-text rendering below its own
+# MIN_EXTRACTED_SIZE (250); that rendering drops heading levels, tables and
+# link targets, so a page that lands between the two thresholds would lose
+# structure instead of just losing its chrome. Bracketing at trafilatura's
+# own floor keeps the full conversion in charge there.
+_MIN_KEEP_RATIO = 0.15
+
+
+def extract_main_content(html: str) -> str:
+ """Extract the main content of a page with trafilatura.
+
+ Returns the extracted markdown, or an empty string when trafilatura finds
+ nothing usable or raises; callers then keep the full conversion.
+ """
+ try:
+ extracted = extract(
+ html,
+ output_format="markdown",
+ include_tables=True,
+ include_links=True,
+ include_formatting=True,
+ # Keep references in Markdown; Go can only resolve/download body
+ # images if their URLs survive this extraction step.
+ include_images=True,
+ )
+ except Exception as exc: # never fail a parse the full conversion can serve
+ logger.warning("HTML main-content extraction failed: %s", exc)
+ return ""
+ return (extracted or "").strip()
+
class HTMLToMarkdownParser(BaseParser):
"""Convert uploaded HTML bytes to Markdown without browser or network access."""
+ def _full_markdown(self, html: str) -> str:
+ """Convert the whole page, the way this parser always did."""
+ return MHTMLParser(
+ file_name=self.file_name,
+ file_type=self.file_type,
+ extract_images=False,
+ ).html_to_markdown(
+ html,
+ strip_internal_links=False,
+ fallback_to_raw_html=False,
+ ) or ""
+
def parse_into_text(self, content: bytes) -> Document:
if not content.strip():
return Document()
@@ -19,16 +80,47 @@ class HTMLToMarkdownParser(BaseParser):
# Inspect the original bytes so BOMs and HTML charset declarations are
# honored before the shared Markdown conversion runs.
html = BeautifulSoup(content, "lxml").decode()
- markdown = MHTMLParser(
- file_name=self.file_name,
- file_type=self.file_type,
- extract_images=False,
- ).html_to_markdown(
- html,
- strip_internal_links=False,
- fallback_to_raw_html=False,
+
+ full = self._full_markdown(html)
+ if not CONFIG.html_main_content:
+ return Document(content=full)
+
+ extracted = extract_main_content(html)
+ # Some short articles are repeated by trafilatura's fallback. Do not
+ # index extra copies or let them inflate the length/retention guards.
+ # Only reject repetitions absent from the full conversion: deliberate
+ # repeated paragraphs in the source must remain valid.
+ blocks = Counter(line.strip() for line in extracted.splitlines())
+ for block, occurrences in blocks.items():
+ if occurrences > 1 and len(block) >= 80 and 0 < full.count(block) < occurrences:
+ logger.info(
+ "HTML main-content extraction duplicated a body block; keeping the full conversion"
+ )
+ return Document(content=full)
+ if len(extracted) < _MIN_EXTRACTED_CHARS:
+ logger.info(
+ "HTML main-content extraction returned %d chars (<%d); "
+ "keeping the full conversion",
+ len(extracted),
+ _MIN_EXTRACTED_CHARS,
+ )
+ return Document(content=full)
+ if full and len(extracted) < len(full) * _MIN_KEEP_RATIO:
+ logger.info(
+ "HTML main-content extraction kept only %d chars of %d; "
+ "keeping the full conversion",
+ len(extracted),
+ len(full),
+ )
+ return Document(content=full)
+
+ logger.info(
+ "HTML main-content extraction applied: kept %d chars, "
+ "full conversion would be %d",
+ len(extracted),
+ len(full),
)
- return Document(content=markdown or "")
+ return Document(content=extracted)
class HTMLParser(PipelineParser):
diff --git a/docreader/testdata/html_main_content/article_page.html b/docreader/testdata/html_main_content/article_page.html
new file mode 100644
index 00000000..c6117093
--- /dev/null
+++ b/docreader/testdata/html_main_content/article_page.html
@@ -0,0 +1,76 @@
+
+
+