feat(docreader): 上传 HTML 只索引正文,与 URL 导入路径对齐(开关,默认关) (#4424)

main
linus-liu-web 2026-10-17 23:24:22 +08:00 committed by GitHub
parent f89a15e61c
commit 44d90beea2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 516 additions and 11 deletions

View File

@ -425,6 +425,9 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# DOCREADER_DOCX_MAX_PAGES=0
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
# 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。
# 默认 false:整页转换,导航/侧栏/页脚也会进索引(与升级前一致)。
# DOCREADER_HTML_MAIN_CONTENT=false
# ----- XLSX / Excel -----
# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112

View File

@ -463,6 +463,9 @@ services:
# ========== 解析通用 ==========
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
# 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。
# 默认 false = 整页转换(导航/侧栏/页脚也会进索引),升级后行为不变。
- DOCREADER_HTML_MAIN_CONTENT=${DOCREADER_HTML_MAIN_CONTENT:-false}
# ========== XLSX / Excel ==========
# 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
# 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的

View File

@ -84,6 +84,10 @@ docreader:
- `DOCREADER_PDF_RENDER_DPI`: 扫描 PDF 渲染 DPI(默认:200)
- `DOCREADER_PDF_JPEG_QUALITY`: 扫描 PDF 输出 JPEG 质量(默认:85,范围会自动限制在 1-95)
### 上传 HTML 的正文抽取
- `DOCREADER_HTML_MAIN_CONTENT`: 上传的 `.html` / `.htm` 文件是否只索引正文(默认:`false`)。开启后用 trafilatura 抽取正文,与 URL 导入路径一致,导航栏 / 侧栏 / 页脚不再进入索引;抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时自动回退整页转换。默认关闭,升级后既有行为不变。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。
### OCR / VLM
DocReader 自身不再内置 OCR 与 VLM 后端。扫描 PDF 会被渲染为 JPEG 图片后交由 Go App 侧调用 OCR/VLM 服务处理,相关配置请参考主项目文档。

View File

@ -61,6 +61,7 @@ class DocReaderConfig:
odl_hybrid_mode: str
odl_hybrid_fallback: bool
odl_markdown_with_html: bool
html_main_content: bool
pdf_render_max_workers: int
pdf_render_parallelism: int
pdf_render_dpi: int
@ -98,6 +99,12 @@ def load_config() -> DocReaderConfig:
odl_markdown_with_html = _get_bool(
["DOCREADER_ODL_MARKDOWN_WITH_HTML"], False
)
# Index only the main content of an uploaded HTML file, the way the URL
# import path already does (trafilatura with a full-page fallback). It is
# off by default on purpose, so upgrading never changes what a knowledge
# base indexes for documents it already ingests: with it off every
# uploaded HTML file is converted in full, exactly as before.
html_main_content = _get_bool(["DOCREADER_HTML_MAIN_CONTENT"], False)
pdf_render_max_workers = _get_int(["DOCREADER_PDF_RENDER_MAX_WORKERS"], 1)
# Intra-document render parallelism: how many worker processes render the
# scanned pages of a SINGLE PDF in parallel. pdfium is not thread-safe, so
@ -140,6 +147,7 @@ def load_config() -> DocReaderConfig:
odl_hybrid_mode=odl_hybrid_mode,
odl_hybrid_fallback=odl_hybrid_fallback,
odl_markdown_with_html=odl_markdown_with_html,
html_main_content=html_main_content,
pdf_render_max_workers=pdf_render_max_workers,
pdf_render_parallelism=pdf_render_parallelism,
pdf_render_dpi=pdf_render_dpi,
@ -168,6 +176,7 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
"DOCREADER_ODL_HYBRID_MODE": cfg.odl_hybrid_mode,
"DOCREADER_ODL_HYBRID_FALLBACK": cfg.odl_hybrid_fallback,
"DOCREADER_ODL_MARKDOWN_WITH_HTML": cfg.odl_markdown_with_html,
"DOCREADER_HTML_MAIN_CONTENT": cfg.html_main_content,
"DOCREADER_PDF_RENDER_MAX_WORKERS": cfg.pdf_render_max_workers,
"DOCREADER_PDF_RENDER_PARALLELISM": cfg.pdf_render_parallelism,
"DOCREADER_PDF_RENDER_DPI": cfg.pdf_render_dpi,

View File

@ -1,17 +1,78 @@
"""Static HTML file parser."""
from bs4 import BeautifulSoup
import logging
from collections import Counter
from bs4 import BeautifulSoup
from trafilatura import extract
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.mhtml_parser import MHTMLParser
logger = logging.getLogger(__name__)
# An uploaded HTML file is converted in full by default, so navigation,
# sidebars and footers are indexed as if they were document body. Setting
# DOCREADER_HTML_MAIN_CONTENT=true indexes only the main content instead, the
# way the URL import path (web_parser.py) already does with trafilatura. It is
# off by default on purpose, so upgrading never changes what a knowledge base
# indexes for documents it already ingests.
#
# Main-content extraction can fail in two ways: the page has no clear article
# body (a form page, a bare link list) or trafilatura misjudges it. Both
# thresholds below keep the full conversion in that case, which is also the
# upstream behavior.
_MIN_EXTRACTED_CHARS = 250
# trafilatura falls back to a plain-text rendering below its own
# MIN_EXTRACTED_SIZE (250); that rendering drops heading levels, tables and
# link targets, so a page that lands between the two thresholds would lose
# structure instead of just losing its chrome. Bracketing at trafilatura's
# own floor keeps the full conversion in charge there.
_MIN_KEEP_RATIO = 0.15
def extract_main_content(html: str) -> str:
"""Extract the main content of a page with trafilatura.
Returns the extracted markdown, or an empty string when trafilatura finds
nothing usable or raises; callers then keep the full conversion.
"""
try:
extracted = extract(
html,
output_format="markdown",
include_tables=True,
include_links=True,
include_formatting=True,
# Keep references in Markdown; Go can only resolve/download body
# images if their URLs survive this extraction step.
include_images=True,
)
except Exception as exc: # never fail a parse the full conversion can serve
logger.warning("HTML main-content extraction failed: %s", exc)
return ""
return (extracted or "").strip()
class HTMLToMarkdownParser(BaseParser):
"""Convert uploaded HTML bytes to Markdown without browser or network access."""
def _full_markdown(self, html: str) -> str:
"""Convert the whole page, the way this parser always did."""
return MHTMLParser(
file_name=self.file_name,
file_type=self.file_type,
extract_images=False,
).html_to_markdown(
html,
strip_internal_links=False,
fallback_to_raw_html=False,
) or ""
def parse_into_text(self, content: bytes) -> Document:
if not content.strip():
return Document()
@ -19,16 +80,47 @@ class HTMLToMarkdownParser(BaseParser):
# Inspect the original bytes so BOMs and HTML charset declarations are
# honored before the shared Markdown conversion runs.
html = BeautifulSoup(content, "lxml").decode()
markdown = MHTMLParser(
file_name=self.file_name,
file_type=self.file_type,
extract_images=False,
).html_to_markdown(
html,
strip_internal_links=False,
fallback_to_raw_html=False,
full = self._full_markdown(html)
if not CONFIG.html_main_content:
return Document(content=full)
extracted = extract_main_content(html)
# Some short articles are repeated by trafilatura's fallback. Do not
# index extra copies or let them inflate the length/retention guards.
# Only reject repetitions absent from the full conversion: deliberate
# repeated paragraphs in the source must remain valid.
blocks = Counter(line.strip() for line in extracted.splitlines())
for block, occurrences in blocks.items():
if occurrences > 1 and len(block) >= 80 and 0 < full.count(block) < occurrences:
logger.info(
"HTML main-content extraction duplicated a body block; keeping the full conversion"
)
return Document(content=full)
if len(extracted) < _MIN_EXTRACTED_CHARS:
logger.info(
"HTML main-content extraction returned %d chars (<%d); "
"keeping the full conversion",
len(extracted),
_MIN_EXTRACTED_CHARS,
)
return Document(content=full)
if full and len(extracted) < len(full) * _MIN_KEEP_RATIO:
logger.info(
"HTML main-content extraction kept only %d chars of %d; "
"keeping the full conversion",
len(extracted),
len(full),
)
return Document(content=full)
logger.info(
"HTML main-content extraction applied: kept %d chars, "
"full conversion would be %d",
len(extracted),
len(full),
)
return Document(content=markdown or "")
return Document(content=extracted)
class HTMLParser(PipelineParser):

View File

@ -0,0 +1,76 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>深入理解 RAG 的召回策略 - 示例技术博客</title>
<link rel="stylesheet" href="/static/site.css">
<script src="/static/analytics.js"></script>
</head>
<body>
<header class="site-header">
<a class="logo" href="/">示例技术博客</a>
<nav class="main-nav">
<ul>
<li><a href="/">首页</a></li>
<li><a href="/pricing">产品定价</a></li>
<li><a href="/docs">文档中心</a></li>
<li><a href="/blog">技术博客</a></li>
<li><a href="/about">关于我们</a></li>
</ul>
</nav>
<form class="site-search" action="/search">
<input type="search" name="q" placeholder="搜索文章">
<button type="submit">搜索</button>
</form>
</header>
<div class="layout">
<main>
<article>
<h1>深入理解 RAG 的召回策略</h1>
<p>检索增强生成的效果上限,几乎完全由召回阶段决定。如果召回的候选里没有包含答案所在的段落,后面再强的生成模型也无能为力。因此调优 RAG 的第一步,永远是先把召回率做上去。</p>
<p>关键词检索依赖倒排索引,对术语、编号、专有名词特别敏感;向量检索依赖语义空间,对同义改写和自然语言提问更友好,但对精确匹配反而容易失手。两者失败的样本分布并不重合。</p>
<p>混合检索把 BM25 的稀疏信号与向量召回融合在一起,常见做法是让两路各取前五十条候选,再用加权求和或倒数排名融合合并成一个列表。融合权重需要按语料特点调,不能直接照搬别人的参数。</p>
<p>重排模型只对前 N 条候选打分,N 通常取二十到一百之间。它比召回模型慢得多,因此不能用来扩大候选集,只能用来把已经召回的候选重新排序。把重排放在召回之后,是成本与效果之间最常见的折中。</p>
<p>分块策略同样影响召回。块太大会把多个主题混在一起,向量的语义被稀释;块太小则一句话被切成两半,答案的上下文不完整。实践中常用的做法是按标题切分,再对超长段落做二次切分,并保留一定的重叠。</p>
<p>评估召回质量不要只看最终答案对不对,而要单独测量命中率与排序位置。把每次线上提问的召回结果落盘,定期人工标注一批,是发现召回退化最直接的手段。</p>
<table>
<thead>
<tr><th>策略</th><th>优点</th><th>代价</th></tr>
</thead>
<tbody>
<tr><td>关键词检索</td><td>术语与编号精确匹配</td><td>同义改写召回不到</td></tr>
<tr><td>向量检索</td><td>语义相近即可召回</td><td>专有名词容易漂移</td></tr>
<tr><td>混合检索</td><td>两路互补,召回率最高</td><td>需要调融合权重</td></tr>
</tbody>
</table>
<p>延伸阅读:<a href="/blog/99">分块器的实现细节</a>。</p>
</article>
</main>
<aside class="sidebar">
<h2>本周热读</h2>
<ul>
<li><a href="/blog/101">向量数据库选型指南</a></li>
<li><a href="/blog/102">从零实现一个分块器</a></li>
<li><a href="/blog/103">重排模型评测实录</a></li>
</ul>
<div class="subscribe">
<p>订阅我们的新闻邮件</p>
<input type="email" placeholder="邮箱地址">
<button type="button">订阅</button>
</div>
</aside>
</div>
<footer class="site-footer">
<p>版权所有 © 2025 示例科技有限公司</p>
<p>京ICP备00000000号-1</p>
<ul>
<li><a href="/privacy">隐私政策</a></li>
<li><a href="/terms">服务条款</a></li>
<li><a href="/contact">联系我们</a></li>
</ul>
</footer>
</body>
</html>

View File

@ -0,0 +1,90 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>发布说明 - 示例开发者门户</title>
</head>
<body>
<header class="topbar">
<nav class="global-nav">
<a href="/">开发者门户首页</a>
<a href="/products">全部产品与服务目录</a>
<a href="/pricing">价格与计费方式说明</a>
<a href="/market">应用市场与合作伙伴</a>
<a href="/support">技术支持与工单系统</a>
<a href="/community">开发者社区与论坛</a>
<a href="/events">线上线下活动日历</a>
<a href="/careers">招聘与团队介绍</a>
<a href="/contact">联系我们与商务合作</a>
<a href="/status">服务健康状态监控页</a>
</nav>
</header>
<div class="layout">
<aside class="left-menu">
<h2>产品文档目录</h2>
<ul>
<li><a href="/docs/a1">快速入门与账号准备</a></li>
<li><a href="/docs/a2">接口鉴权与密钥轮换</a></li>
<li><a href="/docs/a3">数据模型与字段说明</a></li>
<li><a href="/docs/a4">错误码与重试建议</a></li>
<li><a href="/docs/a5">配额限制与扩容申请</a></li>
<li><a href="/docs/a6">回调通知与签名校验</a></li>
<li><a href="/docs/a7">沙箱环境与联调步骤</a></li>
<li><a href="/docs/a8">上线检查清单与灰度</a></li>
</ul>
</aside>
<main>
<article>
<h1>发布说明 2025-09</h1>
<p>本次发布只修复了一个问题:批量导入任务在遇到空文件时不再中断整个批次。当上传的文件列表里混入 0 字节的条目时,任务会跳过该条目并继续处理剩余文件,同时在任务详情里给出跳过原因。升级后无需修改任何配置,历史任务的重试行为也保持一致;如需在升级前临时规避,可以先把空文件从列表里删除再重新提交。</p>
</article>
</main>
<aside class="right-rail">
<h2>相关阅读</h2>
<ul>
<li><a href="/blog/b1">历史发布说明归档</a></li>
<li><a href="/blog/b2">版本兼容性政策说明</a></li>
<li><a href="/blog/b3">已知问题与规避方案</a></li>
<li><a href="/blog/b4">升级步骤与回滚方法</a></li>
</ul>
<div class="tags">
<span>标签:</span><a href="/tag/api">接口</a><a href="/tag/sdk">SDK</a><a href="/tag/quota">配额</a><a href="/tag/webhook">回调</a><a href="/tag/sandbox">沙箱</a>
</div>
</aside>
</div>
<footer class="site-footer">
<div class="about">
<h2>关于示例开发者门户</h2>
<p>示例开发者门户为开发者提供接口文档、SDK 下载、沙箱联调环境与技术支持入口。门户内容按季度更新,历史版本文档保留在归档区,便于对照接口行为变化。文档中的示例代码仅用于说明调用方式,请勿直接用于生产环境。</p>
<p>我们建议在接入前先阅读快速入门与鉴权说明,再按业务场景选择合适的产品组合。如遇文档与接口行为不一致的情况,请以接口实际返回为准,并通过工单系统反馈,我们会在一个工作日内核实并更新文档。</p>
<p>工作日服务时间为 9:30 至 18:30,法定节假日期间仅处理线上故障。商务合作、渠道对接与私有化部署咨询请通过商务合作页面提交,我们会安排对应的解决方案工程师跟进。</p>
<p>门户内的接口文档、SDK 与示例代码均按“现状”提供,示例代码不构成任何形式的质保承诺。第三方商标与品牌名称归各自所有者所有,门户对其内容不承担审核义务,链接仅为方便开发者查阅。</p>
<p>如需转载本站内容,请保留原文链接并注明来源;批量转载、镜像站点与商业用途请先取得书面授权。我们对转载内容的时效性与准确性不做保证,请以本站发布的最新版本为准。</p>
<p>为保障账号安全,门户会在检测到异常登录时要求二次验证,验证方式包括短信验证码、邮箱验证码与已绑定的验证器应用。长期未登录的账号在重新登录时同样会触发二次验证,请提前确认绑定信息仍然可用。</p>
<p>账号注销后,与账号关联的工单记录、沙箱环境与未结算的配额将一并释放且无法恢复。注销前请先导出需要保留的调用日志与账单明细,注销申请提交后会有三个工作日的冷静期,冷静期内可撤回。</p>
<p>门户展示的配额、价格与可用区信息以控制台下单页面为准,页面文案仅作说明用途,不构成报价。涉及跨境数据传输的业务场景,请先确认所在地区的合规要求,并在控制台完成相应的合规声明。</p>
<p>如发现文档中的示例代码存在缺陷,欢迎通过工单或社区反馈;我们会在核实后更新示例并补充变更说明。安全相关问题请勿在公开渠道披露,请通过安全应急响应渠道提交,我们会在收到后一个工作日内确认。</p>
<p>门户的可用性目标为每月 99.9%,计划内维护会提前三个自然日在公告区发布,维护窗口通常安排在北京时间凌晨。因不可抗力、运营商线路调整或第三方服务故障导致的不可用时间,不计入可用性统计。</p>
<p>为持续改进文档质量,我们会统计页面的匿名访问量与搜索关键词,用于判断哪些章节需要补充示例。统计不涉及账号身份信息,也不与工单内容关联;如需退出统计,可在浏览器中启用请勿跟踪选项。</p>
<p>关于计费:按量计费的资源在创建时冻结一小时预估费用,账单在次日凌晨生成;包年包月在到期前七天进入续费提醒状态。资源释放后的按量费用仍会结算到释放当天,历史账单保留二十四个月,可随时导出对账。</p>
<p>关于发票:个人认证账号可申请电子普通发票,企业认证账号可申请增值税专用发票。开票金额以实际支付金额为准,优惠券与代金券抵扣部分不开票;发票申请提交后一般在三个工作日内开具,邮寄纸质发票需要额外时间。</p>
<p>关于数据保留:删除的知识库进入回收站保留三十天,期间可随时恢复;超过保留期后数据会被异步清理且不可恢复。对象存储中的原始文件与向量索引分别清理,两者完成时间可能存在数小时差异。</p>
<p>关于地域与容灾:不同地域之间的数据不自动复制,跨地域访问需要显式配置加速链路。建议在业务侧实现幂等重试,避免网络抖动导致的重复写入;批量任务请使用任务 ID 查询结果,不要依赖单次请求的返回值。</p>
</div>
<ul class="footer-links">
<li><a href="/privacy">隐私政策与数据保护声明</a></li>
<li><a href="/terms">服务条款与使用限制</a></li>
<li><a href="/sla">服务等级协议与赔付标准</a></li>
<li><a href="/security">安全合规与漏洞披露流程</a></li>
<li><a href="/sitemap">网站地图与全部页面索引</a></li>
</ul>
<p>版权所有 © 2025 示例科技有限公司</p>
<p>京ICP备00000000号-1</p>
<p>本网站使用 Cookie 以改善浏览体验,继续访问即表示同意我们的 Cookie 政策。</p>
</footer>
</body>
</html>

View File

@ -0,0 +1,28 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>账号登录</title>
</head>
<body>
<header>
<nav><a href="/">首页</a><a href="/help">帮助中心</a></nav>
</header>
<main>
<h1>账号登录</h1>
<form action="/login" method="post">
<label for="username">用户名</label>
<input id="username" name="username" type="text">
<label for="password">密码</label>
<input id="password" name="password" type="password">
<label for="captcha">验证码</label>
<input id="captcha" name="captcha" type="text">
<button type="submit">登录</button>
<a href="/reset">忘记密码?</a>
</form>
</main>
<footer>
<p>版权所有 © 2025 示例科技有限公司</p>
</footer>
</body>
</html>

View File

@ -0,0 +1,27 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>文档索引</title>
</head>
<body>
<header>
<nav><a href="/">首页</a><a href="/download">下载</a></nav>
</header>
<main>
<h1>文档索引</h1>
<ul>
<li><a href="/docs/01">第一章 快速开始</a></li>
<li><a href="/docs/02">第二章 安装部署</a></li>
<li><a href="/docs/03">第三章 数据接入</a></li>
<li><a href="/docs/04">第四章 检索调优</a></li>
<li><a href="/docs/05">第五章 权限管理</a></li>
<li><a href="/docs/06">第六章 常见问题</a></li>
</ul>
</main>
<footer>
<p>版权所有 © 2025 示例科技有限公司</p>
<p>京ICP备00000000号-1</p>
</footer>
</body>
</html>

View File

@ -0,0 +1,19 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>系统维护通知</title>
</head>
<body>
<header>
<nav><a href="/">首页</a><a href="/status">服务状态</a></nav>
</header>
<main>
<h1>系统维护通知</h1>
<p>本服务将于 10 月 1 日 02:00 至 04:00 维护,期间暂停文档上传与检索。</p>
</main>
<footer>
<p>版权所有 © 2025 示例科技有限公司</p>
</footer>
</body>
</html>

View File

@ -38,6 +38,23 @@ class DocReaderConfigTest(unittest.TestCase):
self.assertIn("DOCREADER_PDF_RENDER_DPI", dumped)
self.assertIn("DOCREADER_PDF_JPEG_QUALITY", dumped)
def test_html_main_content_is_off_by_default(self):
with patch.dict(os.environ, {}, clear=True):
cfg = config.load_config()
self.assertFalse(cfg.html_main_content)
def test_loads_html_main_content_env(self):
with patch.dict(os.environ, {"DOCREADER_HTML_MAIN_CONTENT": "true"}):
cfg = config.load_config()
self.assertTrue(cfg.html_main_content)
def test_dump_config_includes_html_main_content(self):
dumped = config.dump_config()
self.assertIn("DOCREADER_HTML_MAIN_CONTENT", dumped)
if __name__ == "__main__":
unittest.main()

View File

@ -1,10 +1,39 @@
import unittest
from dataclasses import replace
from pathlib import Path
from unittest.mock import patch
from docreader.parser.html_parser import HTMLParser
import docreader.parser.html_parser as html_parser
from docreader.config import CONFIG
from docreader.parser.html_parser import (
_MIN_EXTRACTED_CHARS,
_MIN_KEEP_RATIO,
HTMLParser,
extract_main_content,
)
from docreader.parser.registry import BUILTIN_ENGINE, registry
REPO_ROOT = Path(__file__).resolve().parents[2]
MAIN_CONTENT_FIXTURES = REPO_ROOT / "docreader" / "testdata" / "html_main_content"
def _parse_bytes(
content: bytes, *, main_content: bool, file_name: str = "page.html"
) -> str:
"""Parse HTML through the upload path with the switch forced."""
with patch.object(
html_parser, "CONFIG", replace(CONFIG, html_main_content=main_content)
):
return HTMLParser(file_name=file_name, file_type="html").parse(content).content
def _parse_uploaded_html(name: str, *, main_content: bool) -> str:
"""Parse a fixture through the upload path with the switch forced."""
return _parse_bytes(
(MAIN_CONTENT_FIXTURES / name).read_bytes(),
main_content=main_content,
file_name=name,
)
class HTMLParserTest(unittest.TestCase):
@ -74,5 +103,110 @@ class HTMLParserTest(unittest.TestCase):
self.assertIn("htm", builtin_types)
class HTMLMainContentTest(unittest.TestCase):
"""DOCREADER_HTML_MAIN_CONTENT: opt-in main-content extraction on upload."""
def test_upload_indexes_the_whole_page_by_default(self):
content = _parse_uploaded_html("article_page.html", main_content=False)
# Header nav, sidebar and footer all reach the index, as they always did.
self.assertIn("产品定价", content)
self.assertIn("订阅我们的新闻邮件", content)
self.assertIn("京ICP备00000000号-1", content)
self.assertIn("混合检索", content)
def test_enabled_extraction_drops_page_chrome_and_keeps_the_article(self):
full = _parse_uploaded_html("article_page.html", main_content=False)
content = _parse_uploaded_html("article_page.html", main_content=True)
self.assertNotIn("产品定价", content) # header nav
self.assertNotIn("订阅我们的新闻邮件", content) # sidebar
self.assertNotIn("京ICP备00000000号-1", content) # footer
# The article body survives in full, table and links included.
self.assertIn("# 深入理解 RAG 的召回策略", content)
self.assertIn("混合检索把 BM25 的稀疏信号与向量召回融合在一起", content)
self.assertIn("重排模型只对前 N 条候选打分", content)
self.assertIn("| 策略 | 优点 | 代价 |", content)
self.assertIn("两路互补,召回率最高", content)
self.assertIn("[分块器的实现细节](/blog/99)", content)
self.assertLess(len(content), len(full))
def test_enabled_extraction_preserves_body_image_references(self):
html = (MAIN_CONTENT_FIXTURES / "article_page.html").read_text(encoding="utf-8")
html = html.replace(
"</article>",
'<figure><img src="https://example.com/chart.png" alt="Recall comparison" /></figure>'
'<figure><img src="images/local-chart.png" alt="Local comparison" /></figure>'
'</article>',
)
self.assertIn("https://example.com/chart.png", html)
content = _parse_bytes(html.encode(), main_content=True)
self.assertNotIn("产品定价", content)
self.assertIn("https://example.com/chart.png", content)
self.assertIn("images/local-chart.png", content)
self.assertIn("混合检索", content)
def test_short_article_does_not_index_duplicate_body_text(self):
body = "正文说明知识库如何解析上传文件,以及如何把检索到的内容交给模型生成答案。" * 9
html = f"<html><body><article><h1>知识库解析说明</h1><p>{body}</p></article></body></html>".encode()
content = _parse_bytes(html, main_content=True)
self.assertEqual(content.count(body), 1)
def test_extractor_introduced_repetition_falls_back_but_source_repetition_does_not(self):
body = "A long paragraph about retrieval, citations and document parsing. " * 5
html = f"<article><p>{body}</p></article>".encode()
full = _parse_bytes(html, main_content=False)
repeated = body.strip() + "\n\n" + body.strip()
with patch.object(html_parser, "extract_main_content", return_value=repeated):
self.assertEqual(_parse_bytes(html, main_content=True), full)
original_repeated = f"<article><p>{body}</p><p>{body}</p></article>".encode()
content = _parse_bytes(original_repeated, main_content=True)
self.assertEqual(content.count(body.strip()), 2)
def test_falls_back_to_full_conversion_without_an_article_body(self):
for name, expected in (
("form_page.html", "忘记密码?"),
("link_list_page.html", "第六章 常见问题"),
):
with self.subTest(fixture=name):
full = _parse_uploaded_html(name, main_content=False)
content = _parse_uploaded_html(name, main_content=True)
self.assertEqual(content, full)
self.assertIn(expected, content)
def test_falls_back_when_extraction_keeps_too_little_of_a_long_page(self):
full = _parse_uploaded_html("chrome_heavy_page.html", main_content=False)
content = _parse_uploaded_html("chrome_heavy_page.html", main_content=True)
# The length floor is passed here: what keeps the full conversion is the
# ratio guard against "trafilatura kept only a fragment of a long page".
html = (MAIN_CONTENT_FIXTURES / "chrome_heavy_page.html").read_text(
encoding="utf-8"
)
extracted = extract_main_content(html)
self.assertGreaterEqual(len(extracted), _MIN_EXTRACTED_CHARS)
self.assertLess(len(extracted), len(full) * _MIN_KEEP_RATIO)
self.assertEqual(content, full)
self.assertIn("批量导入任务在遇到空文件时不再中断整个批次", content)
self.assertIn("京ICP备00000000号-1", content)
def test_short_page_is_not_reduced_to_nothing(self):
full = _parse_uploaded_html("short_notice.html", main_content=False)
content = _parse_uploaded_html("short_notice.html", main_content=True)
self.assertEqual(content, full)
self.assertIn(
"本服务将于 10 月 1 日 02:00 至 04:00 维护,期间暂停文档上传与检索。",
content,
)
def test_empty_html_stays_empty(self):
content = _parse_bytes(b"", main_content=True, file_name="empty.html")
self.assertEqual("", content)
if __name__ == "__main__":
unittest.main()

View File

@ -279,6 +279,7 @@ AWS S3 的 `S3_ACCESS_KEY` / `S3_SECRET_KEY` 可以**同时留空**,此时走
| `DOCREADER_PDF_RENDER_DPI` / `DOCREADER_PDF_JPEG_QUALITY` / `DOCREADER_PDF_RENDER_MAX_EDGE` | 200 / 85 / 2000 | PDF 渲染 |
| `DOCREADER_PDF_FORCE_SCANNED` / `DOCREADER_PDF_SCAN_IMAGE_RATIO` / `DOCREADER_PDF_SCAN_MIN_CHARS` | false / 代码默认 | 扫描件判定 |
| `DOCREADER_ODL_HYBRID` / `DOCREADER_ODL_HYBRID_URL` / `DOCREADER_ODL_HYBRID_MODE` / `DOCREADER_ODL_HYBRID_FALLBACK` | off / http://odl-hybrid:5002 / auto / false | OpenDataLoader 混合解析 |
| `DOCREADER_HTML_MAIN_CONTENT` | false | 上传的 HTML 文件只索引正文(与 URL 导入同一抽取策略;抽取不足时回退整页转换) |
| 其余 `DOCREADER_PDF_*`(词距/边栏/隐藏文本/嵌入图/图表区等 20+ 项) | 见 `docker-compose.yml` docreader 段注释 | PDF 版式与抽取精调 |
| `DOCREADER_EXTERNAL_HTTP_PROXY` / `_HTTPS_PROXY` | 空 | docreader 出站抓取代理 |

View File

@ -254,6 +254,7 @@ PPT 系列**没有独立解析器**,由 `MarkitdownParser` 处理,这两个
- `HTMLToMarkdownParser` 先用 `BeautifulSoup(content, "lxml")` 解码原始字节——先看 BOM 与 HTML 内的 charset 声明,再交给统一的 Markdown 转换;
- HTML → Markdown 复用 `MHTMLParser.html_to_markdown()`,但传入 `extract_images=False`(本地 HTML 文件没有 MIME 附件可抽)、`strip_internal_links=False`(保留站内链接)、`fallback_to_raw_html=False`(转换不出内容时返回空而不是塞一整块 ```` ```html ````);
- 正文中通过 `<img src="http://...">` 引用的远程图片由 Go 侧补齐:`internal/infrastructure/docparser/image_resolver.go` 会带 SSRF 校验下载这些远程图片并转存到对象存储,再重写引用,使其与本地上传的图片走同一套 OCR / caption 流程。
- 默认整页转换,导航栏 / 侧栏 / 页脚也会进入索引。设 `DOCREADER_HTML_MAIN_CONTENT=true` 后改用 trafilatura 只抽正文(与 URL 导入路径同一策略),抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时回退整页转换。该开关默认关闭,升级不会改变既有文档的索引内容。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。
#### xmind_parser.py — XMindParser(.xmind 思维导图)
@ -364,6 +365,7 @@ gRPC 响应中不再返回 chunks(`ReadResponse` 没有 chunk 字段);`Exc
| `DOCREADER_ODL_HYBRID_MODE` | `auto` | hybrid 模式参数 |
| `DOCREADER_ODL_HYBRID_FALLBACK` | false | hybrid 失败是否回退 |
| `DOCREADER_ODL_MARKDOWN_WITH_HTML` | false | ODL markdown 允许 HTML |
| `DOCREADER_HTML_MAIN_CONTENT` | false | 上传 HTML 只索引正文(trafilatura 抽取,失败回退整页转换) |
| `DOCREADER_PDF_RENDER_MAX_WORKERS` | 1 | PDF 渲染阶段限流(跨请求) |
| `DOCREADER_PDF_RENDER_PARALLELISM` | `min(4, cpu)` | 单个 PDF 内扫描页渲染的 worker 进程数 |
| `DOCREADER_PDF_RENDER_DPI` | 200 | 扫描页渲染 DPI |