feat(docreader): 上传 HTML 只索引正文,与 URL 导入路径对齐(开关,默认关) (#4424)
parent
f89a15e61c
commit
44d90beea2
|
|
@ -425,6 +425,9 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
|
|||
# DOCREADER_DOCX_MAX_PAGES=0
|
||||
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
|
||||
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
|
||||
# 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。
|
||||
# 默认 false:整页转换,导航/侧栏/页脚也会进索引(与升级前一致)。
|
||||
# DOCREADER_HTML_MAIN_CONTENT=false
|
||||
# ----- XLSX / Excel -----
|
||||
# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
|
||||
# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112
|
||||
|
|
|
|||
|
|
@ -463,6 +463,9 @@ services:
|
|||
# ========== 解析通用 ==========
|
||||
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
|
||||
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
|
||||
# 上传的 HTML 文件是否只索引正文(trafilatura 抽取,抽取不足时回退整页转换)。
|
||||
# 默认 false = 整页转换(导航/侧栏/页脚也会进索引),升级后行为不变。
|
||||
- DOCREADER_HTML_MAIN_CONTENT=${DOCREADER_HTML_MAIN_CONTENT:-false}
|
||||
# ========== XLSX / Excel ==========
|
||||
# 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
|
||||
# 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的
|
||||
|
|
|
|||
|
|
@ -84,6 +84,10 @@ docreader:
|
|||
- `DOCREADER_PDF_RENDER_DPI`: 扫描 PDF 渲染 DPI(默认:200)
|
||||
- `DOCREADER_PDF_JPEG_QUALITY`: 扫描 PDF 输出 JPEG 质量(默认:85,范围会自动限制在 1-95)
|
||||
|
||||
### 上传 HTML 的正文抽取
|
||||
|
||||
- `DOCREADER_HTML_MAIN_CONTENT`: 上传的 `.html` / `.htm` 文件是否只索引正文(默认:`false`)。开启后用 trafilatura 抽取正文,与 URL 导入路径一致,导航栏 / 侧栏 / 页脚不再进入索引;抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时自动回退整页转换。默认关闭,升级后既有行为不变。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。
|
||||
|
||||
### OCR / VLM
|
||||
|
||||
DocReader 自身不再内置 OCR 与 VLM 后端。扫描 PDF 会被渲染为 JPEG 图片后交由 Go App 侧调用 OCR/VLM 服务处理,相关配置请参考主项目文档。
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ class DocReaderConfig:
|
|||
odl_hybrid_mode: str
|
||||
odl_hybrid_fallback: bool
|
||||
odl_markdown_with_html: bool
|
||||
html_main_content: bool
|
||||
pdf_render_max_workers: int
|
||||
pdf_render_parallelism: int
|
||||
pdf_render_dpi: int
|
||||
|
|
@ -98,6 +99,12 @@ def load_config() -> DocReaderConfig:
|
|||
odl_markdown_with_html = _get_bool(
|
||||
["DOCREADER_ODL_MARKDOWN_WITH_HTML"], False
|
||||
)
|
||||
# Index only the main content of an uploaded HTML file, the way the URL
|
||||
# import path already does (trafilatura with a full-page fallback). It is
|
||||
# off by default on purpose, so upgrading never changes what a knowledge
|
||||
# base indexes for documents it already ingests: with it off every
|
||||
# uploaded HTML file is converted in full, exactly as before.
|
||||
html_main_content = _get_bool(["DOCREADER_HTML_MAIN_CONTENT"], False)
|
||||
pdf_render_max_workers = _get_int(["DOCREADER_PDF_RENDER_MAX_WORKERS"], 1)
|
||||
# Intra-document render parallelism: how many worker processes render the
|
||||
# scanned pages of a SINGLE PDF in parallel. pdfium is not thread-safe, so
|
||||
|
|
@ -140,6 +147,7 @@ def load_config() -> DocReaderConfig:
|
|||
odl_hybrid_mode=odl_hybrid_mode,
|
||||
odl_hybrid_fallback=odl_hybrid_fallback,
|
||||
odl_markdown_with_html=odl_markdown_with_html,
|
||||
html_main_content=html_main_content,
|
||||
pdf_render_max_workers=pdf_render_max_workers,
|
||||
pdf_render_parallelism=pdf_render_parallelism,
|
||||
pdf_render_dpi=pdf_render_dpi,
|
||||
|
|
@ -168,6 +176,7 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
|
|||
"DOCREADER_ODL_HYBRID_MODE": cfg.odl_hybrid_mode,
|
||||
"DOCREADER_ODL_HYBRID_FALLBACK": cfg.odl_hybrid_fallback,
|
||||
"DOCREADER_ODL_MARKDOWN_WITH_HTML": cfg.odl_markdown_with_html,
|
||||
"DOCREADER_HTML_MAIN_CONTENT": cfg.html_main_content,
|
||||
"DOCREADER_PDF_RENDER_MAX_WORKERS": cfg.pdf_render_max_workers,
|
||||
"DOCREADER_PDF_RENDER_PARALLELISM": cfg.pdf_render_parallelism,
|
||||
"DOCREADER_PDF_RENDER_DPI": cfg.pdf_render_dpi,
|
||||
|
|
|
|||
|
|
@ -1,17 +1,78 @@
|
|||
"""Static HTML file parser."""
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
import logging
|
||||
from collections import Counter
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from trafilatura import extract
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.mhtml_parser import MHTMLParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# An uploaded HTML file is converted in full by default, so navigation,
|
||||
# sidebars and footers are indexed as if they were document body. Setting
|
||||
# DOCREADER_HTML_MAIN_CONTENT=true indexes only the main content instead, the
|
||||
# way the URL import path (web_parser.py) already does with trafilatura. It is
|
||||
# off by default on purpose, so upgrading never changes what a knowledge base
|
||||
# indexes for documents it already ingests.
|
||||
#
|
||||
# Main-content extraction can fail in two ways: the page has no clear article
|
||||
# body (a form page, a bare link list) or trafilatura misjudges it. Both
|
||||
# thresholds below keep the full conversion in that case, which is also the
|
||||
# upstream behavior.
|
||||
_MIN_EXTRACTED_CHARS = 250
|
||||
# trafilatura falls back to a plain-text rendering below its own
|
||||
# MIN_EXTRACTED_SIZE (250); that rendering drops heading levels, tables and
|
||||
# link targets, so a page that lands between the two thresholds would lose
|
||||
# structure instead of just losing its chrome. Bracketing at trafilatura's
|
||||
# own floor keeps the full conversion in charge there.
|
||||
_MIN_KEEP_RATIO = 0.15
|
||||
|
||||
|
||||
def extract_main_content(html: str) -> str:
|
||||
"""Extract the main content of a page with trafilatura.
|
||||
|
||||
Returns the extracted markdown, or an empty string when trafilatura finds
|
||||
nothing usable or raises; callers then keep the full conversion.
|
||||
"""
|
||||
try:
|
||||
extracted = extract(
|
||||
html,
|
||||
output_format="markdown",
|
||||
include_tables=True,
|
||||
include_links=True,
|
||||
include_formatting=True,
|
||||
# Keep references in Markdown; Go can only resolve/download body
|
||||
# images if their URLs survive this extraction step.
|
||||
include_images=True,
|
||||
)
|
||||
except Exception as exc: # never fail a parse the full conversion can serve
|
||||
logger.warning("HTML main-content extraction failed: %s", exc)
|
||||
return ""
|
||||
return (extracted or "").strip()
|
||||
|
||||
|
||||
class HTMLToMarkdownParser(BaseParser):
|
||||
"""Convert uploaded HTML bytes to Markdown without browser or network access."""
|
||||
|
||||
def _full_markdown(self, html: str) -> str:
|
||||
"""Convert the whole page, the way this parser always did."""
|
||||
return MHTMLParser(
|
||||
file_name=self.file_name,
|
||||
file_type=self.file_type,
|
||||
extract_images=False,
|
||||
).html_to_markdown(
|
||||
html,
|
||||
strip_internal_links=False,
|
||||
fallback_to_raw_html=False,
|
||||
) or ""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
if not content.strip():
|
||||
return Document()
|
||||
|
|
@ -19,16 +80,47 @@ class HTMLToMarkdownParser(BaseParser):
|
|||
# Inspect the original bytes so BOMs and HTML charset declarations are
|
||||
# honored before the shared Markdown conversion runs.
|
||||
html = BeautifulSoup(content, "lxml").decode()
|
||||
markdown = MHTMLParser(
|
||||
file_name=self.file_name,
|
||||
file_type=self.file_type,
|
||||
extract_images=False,
|
||||
).html_to_markdown(
|
||||
html,
|
||||
strip_internal_links=False,
|
||||
fallback_to_raw_html=False,
|
||||
|
||||
full = self._full_markdown(html)
|
||||
if not CONFIG.html_main_content:
|
||||
return Document(content=full)
|
||||
|
||||
extracted = extract_main_content(html)
|
||||
# Some short articles are repeated by trafilatura's fallback. Do not
|
||||
# index extra copies or let them inflate the length/retention guards.
|
||||
# Only reject repetitions absent from the full conversion: deliberate
|
||||
# repeated paragraphs in the source must remain valid.
|
||||
blocks = Counter(line.strip() for line in extracted.splitlines())
|
||||
for block, occurrences in blocks.items():
|
||||
if occurrences > 1 and len(block) >= 80 and 0 < full.count(block) < occurrences:
|
||||
logger.info(
|
||||
"HTML main-content extraction duplicated a body block; keeping the full conversion"
|
||||
)
|
||||
return Document(content=full)
|
||||
if len(extracted) < _MIN_EXTRACTED_CHARS:
|
||||
logger.info(
|
||||
"HTML main-content extraction returned %d chars (<%d); "
|
||||
"keeping the full conversion",
|
||||
len(extracted),
|
||||
_MIN_EXTRACTED_CHARS,
|
||||
)
|
||||
return Document(content=full)
|
||||
if full and len(extracted) < len(full) * _MIN_KEEP_RATIO:
|
||||
logger.info(
|
||||
"HTML main-content extraction kept only %d chars of %d; "
|
||||
"keeping the full conversion",
|
||||
len(extracted),
|
||||
len(full),
|
||||
)
|
||||
return Document(content=full)
|
||||
|
||||
logger.info(
|
||||
"HTML main-content extraction applied: kept %d chars, "
|
||||
"full conversion would be %d",
|
||||
len(extracted),
|
||||
len(full),
|
||||
)
|
||||
return Document(content=markdown or "")
|
||||
return Document(content=extracted)
|
||||
|
||||
|
||||
class HTMLParser(PipelineParser):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>深入理解 RAG 的召回策略 - 示例技术博客</title>
|
||||
<link rel="stylesheet" href="/static/site.css">
|
||||
<script src="/static/analytics.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<header class="site-header">
|
||||
<a class="logo" href="/">示例技术博客</a>
|
||||
<nav class="main-nav">
|
||||
<ul>
|
||||
<li><a href="/">首页</a></li>
|
||||
<li><a href="/pricing">产品定价</a></li>
|
||||
<li><a href="/docs">文档中心</a></li>
|
||||
<li><a href="/blog">技术博客</a></li>
|
||||
<li><a href="/about">关于我们</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
<form class="site-search" action="/search">
|
||||
<input type="search" name="q" placeholder="搜索文章">
|
||||
<button type="submit">搜索</button>
|
||||
</form>
|
||||
</header>
|
||||
|
||||
<div class="layout">
|
||||
<main>
|
||||
<article>
|
||||
<h1>深入理解 RAG 的召回策略</h1>
|
||||
<p>检索增强生成的效果上限,几乎完全由召回阶段决定。如果召回的候选里没有包含答案所在的段落,后面再强的生成模型也无能为力。因此调优 RAG 的第一步,永远是先把召回率做上去。</p>
|
||||
<p>关键词检索依赖倒排索引,对术语、编号、专有名词特别敏感;向量检索依赖语义空间,对同义改写和自然语言提问更友好,但对精确匹配反而容易失手。两者失败的样本分布并不重合。</p>
|
||||
<p>混合检索把 BM25 的稀疏信号与向量召回融合在一起,常见做法是让两路各取前五十条候选,再用加权求和或倒数排名融合合并成一个列表。融合权重需要按语料特点调,不能直接照搬别人的参数。</p>
|
||||
<p>重排模型只对前 N 条候选打分,N 通常取二十到一百之间。它比召回模型慢得多,因此不能用来扩大候选集,只能用来把已经召回的候选重新排序。把重排放在召回之后,是成本与效果之间最常见的折中。</p>
|
||||
<p>分块策略同样影响召回。块太大会把多个主题混在一起,向量的语义被稀释;块太小则一句话被切成两半,答案的上下文不完整。实践中常用的做法是按标题切分,再对超长段落做二次切分,并保留一定的重叠。</p>
|
||||
<p>评估召回质量不要只看最终答案对不对,而要单独测量命中率与排序位置。把每次线上提问的召回结果落盘,定期人工标注一批,是发现召回退化最直接的手段。</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>策略</th><th>优点</th><th>代价</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>关键词检索</td><td>术语与编号精确匹配</td><td>同义改写召回不到</td></tr>
|
||||
<tr><td>向量检索</td><td>语义相近即可召回</td><td>专有名词容易漂移</td></tr>
|
||||
<tr><td>混合检索</td><td>两路互补,召回率最高</td><td>需要调融合权重</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p>延伸阅读:<a href="/blog/99">分块器的实现细节</a>。</p>
|
||||
</article>
|
||||
</main>
|
||||
|
||||
<aside class="sidebar">
|
||||
<h2>本周热读</h2>
|
||||
<ul>
|
||||
<li><a href="/blog/101">向量数据库选型指南</a></li>
|
||||
<li><a href="/blog/102">从零实现一个分块器</a></li>
|
||||
<li><a href="/blog/103">重排模型评测实录</a></li>
|
||||
</ul>
|
||||
<div class="subscribe">
|
||||
<p>订阅我们的新闻邮件</p>
|
||||
<input type="email" placeholder="邮箱地址">
|
||||
<button type="button">订阅</button>
|
||||
</div>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<footer class="site-footer">
|
||||
<p>版权所有 © 2025 示例科技有限公司</p>
|
||||
<p>京ICP备00000000号-1</p>
|
||||
<ul>
|
||||
<li><a href="/privacy">隐私政策</a></li>
|
||||
<li><a href="/terms">服务条款</a></li>
|
||||
<li><a href="/contact">联系我们</a></li>
|
||||
</ul>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>发布说明 - 示例开发者门户</title>
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<nav class="global-nav">
|
||||
<a href="/">开发者门户首页</a>
|
||||
<a href="/products">全部产品与服务目录</a>
|
||||
<a href="/pricing">价格与计费方式说明</a>
|
||||
<a href="/market">应用市场与合作伙伴</a>
|
||||
<a href="/support">技术支持与工单系统</a>
|
||||
<a href="/community">开发者社区与论坛</a>
|
||||
<a href="/events">线上线下活动日历</a>
|
||||
<a href="/careers">招聘与团队介绍</a>
|
||||
<a href="/contact">联系我们与商务合作</a>
|
||||
<a href="/status">服务健康状态监控页</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="layout">
|
||||
<aside class="left-menu">
|
||||
<h2>产品文档目录</h2>
|
||||
<ul>
|
||||
<li><a href="/docs/a1">快速入门与账号准备</a></li>
|
||||
<li><a href="/docs/a2">接口鉴权与密钥轮换</a></li>
|
||||
<li><a href="/docs/a3">数据模型与字段说明</a></li>
|
||||
<li><a href="/docs/a4">错误码与重试建议</a></li>
|
||||
<li><a href="/docs/a5">配额限制与扩容申请</a></li>
|
||||
<li><a href="/docs/a6">回调通知与签名校验</a></li>
|
||||
<li><a href="/docs/a7">沙箱环境与联调步骤</a></li>
|
||||
<li><a href="/docs/a8">上线检查清单与灰度</a></li>
|
||||
</ul>
|
||||
</aside>
|
||||
|
||||
<main>
|
||||
<article>
|
||||
<h1>发布说明 2025-09</h1>
|
||||
<p>本次发布只修复了一个问题:批量导入任务在遇到空文件时不再中断整个批次。当上传的文件列表里混入 0 字节的条目时,任务会跳过该条目并继续处理剩余文件,同时在任务详情里给出跳过原因。升级后无需修改任何配置,历史任务的重试行为也保持一致;如需在升级前临时规避,可以先把空文件从列表里删除再重新提交。</p>
|
||||
</article>
|
||||
</main>
|
||||
|
||||
<aside class="right-rail">
|
||||
<h2>相关阅读</h2>
|
||||
<ul>
|
||||
<li><a href="/blog/b1">历史发布说明归档</a></li>
|
||||
<li><a href="/blog/b2">版本兼容性政策说明</a></li>
|
||||
<li><a href="/blog/b3">已知问题与规避方案</a></li>
|
||||
<li><a href="/blog/b4">升级步骤与回滚方法</a></li>
|
||||
</ul>
|
||||
<div class="tags">
|
||||
<span>标签:</span><a href="/tag/api">接口</a><a href="/tag/sdk">SDK</a><a href="/tag/quota">配额</a><a href="/tag/webhook">回调</a><a href="/tag/sandbox">沙箱</a>
|
||||
</div>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<footer class="site-footer">
|
||||
<div class="about">
|
||||
<h2>关于示例开发者门户</h2>
|
||||
<p>示例开发者门户为开发者提供接口文档、SDK 下载、沙箱联调环境与技术支持入口。门户内容按季度更新,历史版本文档保留在归档区,便于对照接口行为变化。文档中的示例代码仅用于说明调用方式,请勿直接用于生产环境。</p>
|
||||
<p>我们建议在接入前先阅读快速入门与鉴权说明,再按业务场景选择合适的产品组合。如遇文档与接口行为不一致的情况,请以接口实际返回为准,并通过工单系统反馈,我们会在一个工作日内核实并更新文档。</p>
|
||||
<p>工作日服务时间为 9:30 至 18:30,法定节假日期间仅处理线上故障。商务合作、渠道对接与私有化部署咨询请通过商务合作页面提交,我们会安排对应的解决方案工程师跟进。</p>
|
||||
<p>门户内的接口文档、SDK 与示例代码均按“现状”提供,示例代码不构成任何形式的质保承诺。第三方商标与品牌名称归各自所有者所有,门户对其内容不承担审核义务,链接仅为方便开发者查阅。</p>
|
||||
<p>如需转载本站内容,请保留原文链接并注明来源;批量转载、镜像站点与商业用途请先取得书面授权。我们对转载内容的时效性与准确性不做保证,请以本站发布的最新版本为准。</p>
|
||||
<p>为保障账号安全,门户会在检测到异常登录时要求二次验证,验证方式包括短信验证码、邮箱验证码与已绑定的验证器应用。长期未登录的账号在重新登录时同样会触发二次验证,请提前确认绑定信息仍然可用。</p>
|
||||
<p>账号注销后,与账号关联的工单记录、沙箱环境与未结算的配额将一并释放且无法恢复。注销前请先导出需要保留的调用日志与账单明细,注销申请提交后会有三个工作日的冷静期,冷静期内可撤回。</p>
|
||||
<p>门户展示的配额、价格与可用区信息以控制台下单页面为准,页面文案仅作说明用途,不构成报价。涉及跨境数据传输的业务场景,请先确认所在地区的合规要求,并在控制台完成相应的合规声明。</p>
|
||||
<p>如发现文档中的示例代码存在缺陷,欢迎通过工单或社区反馈;我们会在核实后更新示例并补充变更说明。安全相关问题请勿在公开渠道披露,请通过安全应急响应渠道提交,我们会在收到后一个工作日内确认。</p>
|
||||
<p>门户的可用性目标为每月 99.9%,计划内维护会提前三个自然日在公告区发布,维护窗口通常安排在北京时间凌晨。因不可抗力、运营商线路调整或第三方服务故障导致的不可用时间,不计入可用性统计。</p>
|
||||
<p>为持续改进文档质量,我们会统计页面的匿名访问量与搜索关键词,用于判断哪些章节需要补充示例。统计不涉及账号身份信息,也不与工单内容关联;如需退出统计,可在浏览器中启用请勿跟踪选项。</p>
|
||||
<p>关于计费:按量计费的资源在创建时冻结一小时预估费用,账单在次日凌晨生成;包年包月在到期前七天进入续费提醒状态。资源释放后的按量费用仍会结算到释放当天,历史账单保留二十四个月,可随时导出对账。</p>
|
||||
<p>关于发票:个人认证账号可申请电子普通发票,企业认证账号可申请增值税专用发票。开票金额以实际支付金额为准,优惠券与代金券抵扣部分不开票;发票申请提交后一般在三个工作日内开具,邮寄纸质发票需要额外时间。</p>
|
||||
<p>关于数据保留:删除的知识库进入回收站保留三十天,期间可随时恢复;超过保留期后数据会被异步清理且不可恢复。对象存储中的原始文件与向量索引分别清理,两者完成时间可能存在数小时差异。</p>
|
||||
<p>关于地域与容灾:不同地域之间的数据不自动复制,跨地域访问需要显式配置加速链路。建议在业务侧实现幂等重试,避免网络抖动导致的重复写入;批量任务请使用任务 ID 查询结果,不要依赖单次请求的返回值。</p>
|
||||
</div>
|
||||
<ul class="footer-links">
|
||||
<li><a href="/privacy">隐私政策与数据保护声明</a></li>
|
||||
<li><a href="/terms">服务条款与使用限制</a></li>
|
||||
<li><a href="/sla">服务等级协议与赔付标准</a></li>
|
||||
<li><a href="/security">安全合规与漏洞披露流程</a></li>
|
||||
<li><a href="/sitemap">网站地图与全部页面索引</a></li>
|
||||
</ul>
|
||||
<p>版权所有 © 2025 示例科技有限公司</p>
|
||||
<p>京ICP备00000000号-1</p>
|
||||
<p>本网站使用 Cookie 以改善浏览体验,继续访问即表示同意我们的 Cookie 政策。</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>账号登录</title>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav><a href="/">首页</a><a href="/help">帮助中心</a></nav>
|
||||
</header>
|
||||
<main>
|
||||
<h1>账号登录</h1>
|
||||
<form action="/login" method="post">
|
||||
<label for="username">用户名</label>
|
||||
<input id="username" name="username" type="text">
|
||||
<label for="password">密码</label>
|
||||
<input id="password" name="password" type="password">
|
||||
<label for="captcha">验证码</label>
|
||||
<input id="captcha" name="captcha" type="text">
|
||||
<button type="submit">登录</button>
|
||||
<a href="/reset">忘记密码?</a>
|
||||
</form>
|
||||
</main>
|
||||
<footer>
|
||||
<p>版权所有 © 2025 示例科技有限公司</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>文档索引</title>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav><a href="/">首页</a><a href="/download">下载</a></nav>
|
||||
</header>
|
||||
<main>
|
||||
<h1>文档索引</h1>
|
||||
<ul>
|
||||
<li><a href="/docs/01">第一章 快速开始</a></li>
|
||||
<li><a href="/docs/02">第二章 安装部署</a></li>
|
||||
<li><a href="/docs/03">第三章 数据接入</a></li>
|
||||
<li><a href="/docs/04">第四章 检索调优</a></li>
|
||||
<li><a href="/docs/05">第五章 权限管理</a></li>
|
||||
<li><a href="/docs/06">第六章 常见问题</a></li>
|
||||
</ul>
|
||||
</main>
|
||||
<footer>
|
||||
<p>版权所有 © 2025 示例科技有限公司</p>
|
||||
<p>京ICP备00000000号-1</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>系统维护通知</title>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<nav><a href="/">首页</a><a href="/status">服务状态</a></nav>
|
||||
</header>
|
||||
<main>
|
||||
<h1>系统维护通知</h1>
|
||||
<p>本服务将于 10 月 1 日 02:00 至 04:00 维护,期间暂停文档上传与检索。</p>
|
||||
</main>
|
||||
<footer>
|
||||
<p>版权所有 © 2025 示例科技有限公司</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -38,6 +38,23 @@ class DocReaderConfigTest(unittest.TestCase):
|
|||
self.assertIn("DOCREADER_PDF_RENDER_DPI", dumped)
|
||||
self.assertIn("DOCREADER_PDF_JPEG_QUALITY", dumped)
|
||||
|
||||
def test_html_main_content_is_off_by_default(self):
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
cfg = config.load_config()
|
||||
|
||||
self.assertFalse(cfg.html_main_content)
|
||||
|
||||
def test_loads_html_main_content_env(self):
|
||||
with patch.dict(os.environ, {"DOCREADER_HTML_MAIN_CONTENT": "true"}):
|
||||
cfg = config.load_config()
|
||||
|
||||
self.assertTrue(cfg.html_main_content)
|
||||
|
||||
def test_dump_config_includes_html_main_content(self):
|
||||
dumped = config.dump_config()
|
||||
|
||||
self.assertIn("DOCREADER_HTML_MAIN_CONTENT", dumped)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,10 +1,39 @@
|
|||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from docreader.parser.html_parser import HTMLParser
|
||||
import docreader.parser.html_parser as html_parser
|
||||
from docreader.config import CONFIG
|
||||
from docreader.parser.html_parser import (
|
||||
_MIN_EXTRACTED_CHARS,
|
||||
_MIN_KEEP_RATIO,
|
||||
HTMLParser,
|
||||
extract_main_content,
|
||||
)
|
||||
from docreader.parser.registry import BUILTIN_ENGINE, registry
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
MAIN_CONTENT_FIXTURES = REPO_ROOT / "docreader" / "testdata" / "html_main_content"
|
||||
|
||||
|
||||
def _parse_bytes(
|
||||
content: bytes, *, main_content: bool, file_name: str = "page.html"
|
||||
) -> str:
|
||||
"""Parse HTML through the upload path with the switch forced."""
|
||||
with patch.object(
|
||||
html_parser, "CONFIG", replace(CONFIG, html_main_content=main_content)
|
||||
):
|
||||
return HTMLParser(file_name=file_name, file_type="html").parse(content).content
|
||||
|
||||
|
||||
def _parse_uploaded_html(name: str, *, main_content: bool) -> str:
|
||||
"""Parse a fixture through the upload path with the switch forced."""
|
||||
return _parse_bytes(
|
||||
(MAIN_CONTENT_FIXTURES / name).read_bytes(),
|
||||
main_content=main_content,
|
||||
file_name=name,
|
||||
)
|
||||
|
||||
|
||||
class HTMLParserTest(unittest.TestCase):
|
||||
|
|
@ -74,5 +103,110 @@ class HTMLParserTest(unittest.TestCase):
|
|||
self.assertIn("htm", builtin_types)
|
||||
|
||||
|
||||
class HTMLMainContentTest(unittest.TestCase):
|
||||
"""DOCREADER_HTML_MAIN_CONTENT: opt-in main-content extraction on upload."""
|
||||
|
||||
def test_upload_indexes_the_whole_page_by_default(self):
|
||||
content = _parse_uploaded_html("article_page.html", main_content=False)
|
||||
|
||||
# Header nav, sidebar and footer all reach the index, as they always did.
|
||||
self.assertIn("产品定价", content)
|
||||
self.assertIn("订阅我们的新闻邮件", content)
|
||||
self.assertIn("京ICP备00000000号-1", content)
|
||||
self.assertIn("混合检索", content)
|
||||
|
||||
def test_enabled_extraction_drops_page_chrome_and_keeps_the_article(self):
|
||||
full = _parse_uploaded_html("article_page.html", main_content=False)
|
||||
content = _parse_uploaded_html("article_page.html", main_content=True)
|
||||
|
||||
self.assertNotIn("产品定价", content) # header nav
|
||||
self.assertNotIn("订阅我们的新闻邮件", content) # sidebar
|
||||
self.assertNotIn("京ICP备00000000号-1", content) # footer
|
||||
# The article body survives in full, table and links included.
|
||||
self.assertIn("# 深入理解 RAG 的召回策略", content)
|
||||
self.assertIn("混合检索把 BM25 的稀疏信号与向量召回融合在一起", content)
|
||||
self.assertIn("重排模型只对前 N 条候选打分", content)
|
||||
self.assertIn("| 策略 | 优点 | 代价 |", content)
|
||||
self.assertIn("两路互补,召回率最高", content)
|
||||
self.assertIn("[分块器的实现细节](/blog/99)", content)
|
||||
self.assertLess(len(content), len(full))
|
||||
|
||||
def test_enabled_extraction_preserves_body_image_references(self):
|
||||
html = (MAIN_CONTENT_FIXTURES / "article_page.html").read_text(encoding="utf-8")
|
||||
html = html.replace(
|
||||
"</article>",
|
||||
'<figure><img src="https://example.com/chart.png" alt="Recall comparison" /></figure>'
|
||||
'<figure><img src="images/local-chart.png" alt="Local comparison" /></figure>'
|
||||
'</article>',
|
||||
)
|
||||
self.assertIn("https://example.com/chart.png", html)
|
||||
content = _parse_bytes(html.encode(), main_content=True)
|
||||
self.assertNotIn("产品定价", content)
|
||||
self.assertIn("https://example.com/chart.png", content)
|
||||
self.assertIn("images/local-chart.png", content)
|
||||
self.assertIn("混合检索", content)
|
||||
|
||||
def test_short_article_does_not_index_duplicate_body_text(self):
|
||||
body = "正文说明知识库如何解析上传文件,以及如何把检索到的内容交给模型生成答案。" * 9
|
||||
html = f"<html><body><article><h1>知识库解析说明</h1><p>{body}</p></article></body></html>".encode()
|
||||
content = _parse_bytes(html, main_content=True)
|
||||
self.assertEqual(content.count(body), 1)
|
||||
|
||||
def test_extractor_introduced_repetition_falls_back_but_source_repetition_does_not(self):
|
||||
body = "A long paragraph about retrieval, citations and document parsing. " * 5
|
||||
html = f"<article><p>{body}</p></article>".encode()
|
||||
full = _parse_bytes(html, main_content=False)
|
||||
repeated = body.strip() + "\n\n" + body.strip()
|
||||
with patch.object(html_parser, "extract_main_content", return_value=repeated):
|
||||
self.assertEqual(_parse_bytes(html, main_content=True), full)
|
||||
original_repeated = f"<article><p>{body}</p><p>{body}</p></article>".encode()
|
||||
content = _parse_bytes(original_repeated, main_content=True)
|
||||
self.assertEqual(content.count(body.strip()), 2)
|
||||
|
||||
def test_falls_back_to_full_conversion_without_an_article_body(self):
|
||||
for name, expected in (
|
||||
("form_page.html", "忘记密码?"),
|
||||
("link_list_page.html", "第六章 常见问题"),
|
||||
):
|
||||
with self.subTest(fixture=name):
|
||||
full = _parse_uploaded_html(name, main_content=False)
|
||||
content = _parse_uploaded_html(name, main_content=True)
|
||||
|
||||
self.assertEqual(content, full)
|
||||
self.assertIn(expected, content)
|
||||
|
||||
def test_falls_back_when_extraction_keeps_too_little_of_a_long_page(self):
|
||||
full = _parse_uploaded_html("chrome_heavy_page.html", main_content=False)
|
||||
content = _parse_uploaded_html("chrome_heavy_page.html", main_content=True)
|
||||
|
||||
# The length floor is passed here: what keeps the full conversion is the
|
||||
# ratio guard against "trafilatura kept only a fragment of a long page".
|
||||
html = (MAIN_CONTENT_FIXTURES / "chrome_heavy_page.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
extracted = extract_main_content(html)
|
||||
self.assertGreaterEqual(len(extracted), _MIN_EXTRACTED_CHARS)
|
||||
self.assertLess(len(extracted), len(full) * _MIN_KEEP_RATIO)
|
||||
|
||||
self.assertEqual(content, full)
|
||||
self.assertIn("批量导入任务在遇到空文件时不再中断整个批次", content)
|
||||
self.assertIn("京ICP备00000000号-1", content)
|
||||
|
||||
def test_short_page_is_not_reduced_to_nothing(self):
|
||||
full = _parse_uploaded_html("short_notice.html", main_content=False)
|
||||
content = _parse_uploaded_html("short_notice.html", main_content=True)
|
||||
|
||||
self.assertEqual(content, full)
|
||||
self.assertIn(
|
||||
"本服务将于 10 月 1 日 02:00 至 04:00 维护,期间暂停文档上传与检索。",
|
||||
content,
|
||||
)
|
||||
|
||||
def test_empty_html_stays_empty(self):
|
||||
content = _parse_bytes(b"", main_content=True, file_name="empty.html")
|
||||
|
||||
self.assertEqual("", content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -279,6 +279,7 @@ AWS S3 的 `S3_ACCESS_KEY` / `S3_SECRET_KEY` 可以**同时留空**,此时走
|
|||
| `DOCREADER_PDF_RENDER_DPI` / `DOCREADER_PDF_JPEG_QUALITY` / `DOCREADER_PDF_RENDER_MAX_EDGE` | 200 / 85 / 2000 | PDF 渲染 |
|
||||
| `DOCREADER_PDF_FORCE_SCANNED` / `DOCREADER_PDF_SCAN_IMAGE_RATIO` / `DOCREADER_PDF_SCAN_MIN_CHARS` | false / 代码默认 | 扫描件判定 |
|
||||
| `DOCREADER_ODL_HYBRID` / `DOCREADER_ODL_HYBRID_URL` / `DOCREADER_ODL_HYBRID_MODE` / `DOCREADER_ODL_HYBRID_FALLBACK` | off / http://odl-hybrid:5002 / auto / false | OpenDataLoader 混合解析 |
|
||||
| `DOCREADER_HTML_MAIN_CONTENT` | false | 上传的 HTML 文件只索引正文(与 URL 导入同一抽取策略;抽取不足时回退整页转换) |
|
||||
| 其余 `DOCREADER_PDF_*`(词距/边栏/隐藏文本/嵌入图/图表区等 20+ 项) | 见 `docker-compose.yml` docreader 段注释 | PDF 版式与抽取精调 |
|
||||
| `DOCREADER_EXTERNAL_HTTP_PROXY` / `_HTTPS_PROXY` | 空 | docreader 出站抓取代理 |
|
||||
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ PPT 系列**没有独立解析器**,由 `MarkitdownParser` 处理,这两个
|
|||
- `HTMLToMarkdownParser` 先用 `BeautifulSoup(content, "lxml")` 解码原始字节——先看 BOM 与 HTML 内的 charset 声明,再交给统一的 Markdown 转换;
|
||||
- HTML → Markdown 复用 `MHTMLParser.html_to_markdown()`,但传入 `extract_images=False`(本地 HTML 文件没有 MIME 附件可抽)、`strip_internal_links=False`(保留站内链接)、`fallback_to_raw_html=False`(转换不出内容时返回空而不是塞一整块 ```` ```html ````);
|
||||
- 正文中通过 `<img src="http://...">` 引用的远程图片由 Go 侧补齐:`internal/infrastructure/docparser/image_resolver.go` 会带 SSRF 校验下载这些远程图片并转存到对象存储,再重写引用,使其与本地上传的图片走同一套 OCR / caption 流程。
|
||||
- 默认整页转换,导航栏 / 侧栏 / 页脚也会进入索引。设 `DOCREADER_HTML_MAIN_CONTENT=true` 后改用 trafilatura 只抽正文(与 URL 导入路径同一策略),抽取结果过短(< 250 字符)、不足整页转换的 15%,或抽取器额外重复正文段落时回退整页转换。该开关默认关闭,升级不会改变既有文档的索引内容。 正文图片引用会保留在 Markdown 中,供 Go 侧解析;Python 侧不会下载远程图片。
|
||||
|
||||
#### xmind_parser.py — XMindParser(.xmind 思维导图)
|
||||
|
||||
|
|
@ -364,6 +365,7 @@ gRPC 响应中不再返回 chunks(`ReadResponse` 没有 chunk 字段);`Exc
|
|||
| `DOCREADER_ODL_HYBRID_MODE` | `auto` | hybrid 模式参数 |
|
||||
| `DOCREADER_ODL_HYBRID_FALLBACK` | false | hybrid 失败是否回退 |
|
||||
| `DOCREADER_ODL_MARKDOWN_WITH_HTML` | false | ODL markdown 允许 HTML |
|
||||
| `DOCREADER_HTML_MAIN_CONTENT` | false | 上传 HTML 只索引正文(trafilatura 抽取,失败回退整页转换) |
|
||||
| `DOCREADER_PDF_RENDER_MAX_WORKERS` | 1 | PDF 渲染阶段限流(跨请求) |
|
||||
| `DOCREADER_PDF_RENDER_PARALLELISM` | `min(4, cpu)` | 单个 PDF 内扫描页渲染的 worker 进程数 |
|
||||
| `DOCREADER_PDF_RENDER_DPI` | 200 | 扫描页渲染 DPI |
|
||||
|
|
|
|||
Loading…
Reference in New Issue