fix(docparser): make the MinerU parse timeouts configurable

Self-hosted MinerU capped every parse at a hardcoded 1000s, which fired
before WEKNORA_DOCREADER_CALL_TIMEOUT and WEKNORA_DOCUMENT_PROCESS_TIMEOUT
could matter, so large PDFs always failed at 16m40s (#3091). MinerU Cloud
had the same problem with a hardcoded 600s polling deadline.

- WEKNORA_MINERU_TIMEOUT (default 1000s) bounds a self-hosted parse: the
  whole V1 job, or the legacy /file_parse request.
- WEKNORA_MINERU_CLOUD_TIMEOUT (default 600s) bounds how long a MinerU
  Cloud batch is polled.
- Both share the env parsing added for WEKNORA_PADDLEOCR_VL_TIMEOUT in
  #3537, now in helpers.go: invalid or non-positive values log a warning
  and fall back to the default.
main
wizardchen 2026-09-25 18:24:18 +08:00 committed by lyingbug
parent 8a75bece6d
commit 27b0c84bfd
9 changed files with 169 additions and 28 deletions

View File

@ -513,6 +513,11 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# 自建 PaddleOCR-VL HTTP 请求超时(正数 Go duration,默认 1000s;无效值回退默认值)。
# 大文件示例:本项 90m、DocReader 100m、单文档任务 2h,外层超时需留余量。
# WEKNORA_PADDLEOCR_VL_TIMEOUT=1000s
# 自建 MinerU 单次解析超时(正数 Go duration,默认 1000s;无效值回退默认值)。
# 超大 PDF 同样需要外层留余量,例如本项 90m、DocReader 100m、单文档任务 2h。
# WEKNORA_MINERU_TIMEOUT=1000s
# MinerU 云端(mineru.net)轮询解析结果的最长时间(正数 Go duration,默认 600s)。
# WEKNORA_MINERU_CLOUD_TIMEOUT=600s
# 知识库巡检回收(回收卡在 processing 的脏数据;false 禁用,不建议)。
# WEKNORA_HOUSEKEEPING_ENABLED=true
# 聊天附件解析产物保留时长(小时,默认 24)。

View File

@ -332,6 +332,10 @@ services:
- WEKNORA_DOCREADER_CALL_TIMEOUT=${WEKNORA_DOCREADER_CALL_TIMEOUT:-}
# 自建 PaddleOCR-VL HTTP 请求超时(Go duration,默认 1000s)
- WEKNORA_PADDLEOCR_VL_TIMEOUT=${WEKNORA_PADDLEOCR_VL_TIMEOUT:-}
# 自建 MinerU 单次解析超时(Go duration,默认 1000s)
- WEKNORA_MINERU_TIMEOUT=${WEKNORA_MINERU_TIMEOUT:-}
# MinerU 云端轮询解析结果的最长时间(Go duration,默认 600s)
- WEKNORA_MINERU_CLOUD_TIMEOUT=${WEKNORA_MINERU_CLOUD_TIMEOUT:-}
# 知识库巡检回收(回收卡在 processing 的脏数据;false 禁用,不建议)
- WEKNORA_HOUSEKEEPING_ENABLED=${WEKNORA_HOUSEKEEPING_ENABLED:-}
- WEKNORA_CHAT_ATTACHMENT_TTL_HOURS=${WEKNORA_CHAT_ATTACHMENT_TTL_HOURS:-24}

View File

@ -3,6 +3,7 @@ package docparser
import (
"context"
"fmt"
"os"
"sort"
"strings"
"time"
@ -28,6 +29,23 @@ func parseBoolOr(val string, fallback bool) bool {
return val == "true" || val == "1" || val == "yes"
}
// requestTimeoutFromEnv reads a positive Go duration from env, falling back to
// fallback when the variable is unset, invalid or not positive. It bounds a
// self-hosted engine's request; the caller's context may impose an earlier
// deadline.
func requestTimeoutFromEnv(env string, fallback time.Duration) time.Duration {
value := strings.TrimSpace(os.Getenv(env))
if value == "" {
return fallback
}
timeout, err := time.ParseDuration(value)
if err != nil || timeout <= 0 {
logger.Warnf(context.Background(), "Invalid %s %q; using %s", env, value, fallback)
return fallback
}
return timeout
}
// firstNonEmpty returns the first non-empty string, or "" if all are empty.
func firstNonEmpty(vals ...string) string {
for _, v := range vals {

View File

@ -23,6 +23,8 @@ import (
const (
defaultPollInterval = 3 * time.Second
// defaultCloudTimeout bounds how long one batch is polled;
// WEKNORA_MINERU_CLOUD_TIMEOUT overrides it for documents that take longer.
defaultCloudTimeout = 600 * time.Second
defaultBaseURL = "https://mineru.net/api/v4"
)
@ -32,6 +34,8 @@ const (
type MinerUCloudReader struct {
apiKey string
baseURL string
timeout time.Duration
pollInterval time.Duration
model string
formulaEnable bool
tableEnable bool
@ -44,6 +48,8 @@ func NewMinerUCloudReader(overrides map[string]string) *MinerUCloudReader {
return &MinerUCloudReader{
apiKey: strings.TrimSpace(overrides["mineru_api_key"]),
baseURL: defaultBaseURL,
timeout: requestTimeoutFromEnv("WEKNORA_MINERU_CLOUD_TIMEOUT", defaultCloudTimeout),
pollInterval: defaultPollInterval,
model: stringOr(overrides["mineru_cloud_model"], "pipeline"),
formulaEnable: parseBoolOr(overrides["mineru_cloud_enable_formula"], true),
tableEnable: parseBoolOr(overrides["mineru_cloud_enable_table"], true),
@ -211,7 +217,7 @@ type extractResultItem struct {
func (c *MinerUCloudReader) pollBatchResult(
ctx context.Context, batchID string,
) (string, []types.ImageRef, []byte, error) {
deadline := time.Now().Add(defaultCloudTimeout)
deadline := time.Now().Add(c.timeout)
pollCount := 0
headers := map[string]string{
"Authorization": "Bearer " + c.apiKey,
@ -230,7 +236,7 @@ func (c *MinerUCloudReader) pollBatchResult(
items, err := c.fetchBatchStatus(ctx, batchID, headers)
if err != nil {
logger.Errorf(context.Background(), "[MinerUCloud] poll #%d failed: %v", pollCount, err)
sleepCtx(ctx, defaultPollInterval)
sleepCtx(ctx, c.pollInterval)
continue
}
@ -238,7 +244,7 @@ func (c *MinerUCloudReader) pollBatchResult(
if pollCount <= 3 || pollCount%10 == 0 {
logger.Infof(context.Background(), "[MinerUCloud] poll #%d: extract_result empty, retrying", pollCount)
}
sleepCtx(ctx, defaultPollInterval)
sleepCtx(ctx, c.pollInterval)
continue
}
@ -258,10 +264,10 @@ func (c *MinerUCloudReader) pollBatchResult(
return c.extractDoneResult(ctx, &item)
}
sleepCtx(ctx, defaultPollInterval)
sleepCtx(ctx, c.pollInterval)
}
return "", nil, nil, fmt.Errorf("MinerU Cloud task timed out after %d polls", pollCount)
return "", nil, nil, fmt.Errorf("MinerU Cloud task timed out after %s (%d polls)", c.timeout, pollCount)
}
func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string, headers map[string]string) ([]extractResultItem, error) {

View File

@ -22,7 +22,9 @@ import (
"github.com/Tencent/WeKnora/internal/utils"
)
const mineruTimeout = 1000 * time.Second // large docs can take a while
// defaultMinerUTimeout bounds one self-hosted parse; WEKNORA_MINERU_TIMEOUT
// overrides it for documents that take longer.
const defaultMinerUTimeout = 1000 * time.Second // large docs can take a while
var (
b64DataURIPattern = regexp.MustCompile(`^data:image/(\w+);base64,(.+)$`)
@ -35,6 +37,7 @@ var (
// in-place MinerU upgrade needs no configuration change.
type MinerUReader struct {
endpoint string
timeout time.Duration
apiKey string // V1 only: the server's --api-key
tier string // V1 only: flash, basic, standard, advanced; "" = server default
// Legacy (<= 3.x) only; MinerU 4.0 dropped these request parameters.
@ -57,6 +60,7 @@ func NewMinerUReader(overrides map[string]string) *MinerUReader {
c := &MinerUReader{
endpoint: strings.TrimRight(overrides["mineru_endpoint"], "/"),
timeout: requestTimeoutFromEnv("WEKNORA_MINERU_TIMEOUT", defaultMinerUTimeout),
apiKey: strings.TrimSpace(overrides["mineru_server_api_key"]),
tier: resolveMinerUTier(overrides["mineru_tier"]),
backend: stringOr(overrides["mineru_model"], "pipeline"),
@ -138,7 +142,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
}
func (c *MinerUReader) readV1(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
client := newMinerUV1Client(c.endpoint, c.apiKey, mineruTimeout)
client := newMinerUV1Client(c.endpoint, c.apiKey, c.timeout)
mdContent, imageRefs, contentList, err := client.ParseWithLayout(ctx, req.FileContent,
minerUUploadFileName(req.FileName, req.FileType),
minerUV1ParseOptions{Tier: c.tier, OCRMode: c.parseMethod})
@ -310,7 +314,7 @@ func (c *MinerUReader) callFileParse(
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
Timeout: mineruTimeout,
Timeout: c.timeout,
MaxRedirects: 5,
})
resp, err := client.Do(httpReq)

View File

@ -0,0 +1,117 @@
package docparser
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/Tencent/WeKnora/internal/types"
)
func TestMinerUTimeoutConfig(t *testing.T) {
for _, tt := range []struct {
name, value string
want time.Duration
}{
{"empty", "", defaultMinerUTimeout},
{"whitespace", " ", defaultMinerUTimeout},
{"minutes", "90m", 90 * time.Minute},
{"trimmed", " 5400s ", 90 * time.Minute},
{"invalid", "invalid", defaultMinerUTimeout},
{"unitless", "5400", defaultMinerUTimeout},
{"zero", "0s", defaultMinerUTimeout},
{"negative", "-1s", defaultMinerUTimeout},
} {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("WEKNORA_MINERU_TIMEOUT", tt.value)
assert.Equal(t, tt.want, NewMinerUReader(nil).timeout)
})
}
}
// The configured timeout, not the old hardcoded 1000s, must end a parse on
// both protocols, and it must fire before the caller's own deadline.
func TestMinerUReaderHonorsConfiguredTimeout(t *testing.T) {
allowLoopbackSSRF(t)
req := &types.ReadRequest{FileContent: []byte("%PDF"), FileName: "a.pdf", FileType: "pdf"}
t.Run("v1", func(t *testing.T) {
t.Setenv("WEKNORA_MINERU_TIMEOUT", "200ms")
statuses := make([]string, 1000)
for i := range statuses {
statuses[i] = "running"
}
fake := &fakeMinerUV1{t: t, pollStatuses: statuses}
server := httptest.NewServer(fake.handler())
defer server.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err := NewMinerUReader(map[string]string{"mineru_endpoint": server.URL}).Read(ctx, req)
require.Error(t, err)
assert.ErrorIs(t, err, context.DeadlineExceeded)
assert.NoError(t, ctx.Err(), "the configured timeout, not the caller's deadline, should stop the parse")
assert.True(t, fake.canceled, "abandoned job should be canceled on the server")
})
t.Run("legacy", func(t *testing.T) {
t.Setenv("WEKNORA_MINERU_TIMEOUT", "100ms")
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/health" {
http.NotFound(w, r)
return
}
_, _ = io.Copy(io.Discard, r.Body)
select {
case <-r.Context().Done():
case <-release:
}
}))
defer server.Close()
defer close(release)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err := NewMinerUReader(map[string]string{"mineru_endpoint": server.URL}).Read(ctx, req)
require.Error(t, err)
assert.ErrorIs(t, err, context.DeadlineExceeded)
assert.NoError(t, ctx.Err(), "the configured timeout, not the caller's deadline, should stop the parse")
})
}
func TestMinerUCloudTimeoutConfig(t *testing.T) {
t.Setenv("WEKNORA_MINERU_CLOUD_TIMEOUT", "")
assert.Equal(t, defaultCloudTimeout, NewMinerUCloudReader(nil).timeout)
t.Setenv("WEKNORA_MINERU_CLOUD_TIMEOUT", "45m")
assert.Equal(t, 45*time.Minute, NewMinerUCloudReader(nil).timeout)
t.Setenv("WEKNORA_MINERU_CLOUD_TIMEOUT", "0s")
assert.Equal(t, defaultCloudTimeout, NewMinerUCloudReader(nil).timeout)
}
func TestMinerUCloudPollStopsAtConfiguredTimeout(t *testing.T) {
allowLoopbackSSRF(t)
t.Setenv("WEKNORA_MINERU_CLOUD_TIMEOUT", "200ms")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"code":0,"data":{"extract_result":[{"state":"running"}]}}`)
}))
defer server.Close()
reader := NewMinerUCloudReader(nil)
reader.baseURL = server.URL
reader.pollInterval = 10 * time.Millisecond
start := time.Now()
_, _, _, err := reader.pollBatchResult(context.Background(), "batch-1")
require.Error(t, err)
assert.Contains(t, err.Error(), fmt.Sprintf("timed out after %s", 200*time.Millisecond))
assert.Less(t, time.Since(start), 5*time.Second)
}

View File

@ -9,7 +9,6 @@ import (
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"time"
@ -37,29 +36,13 @@ type PaddleOCRVLReader struct {
// NewPaddleOCRVLReader creates a reader from ParserEngineOverrides.
func NewPaddleOCRVLReader(overrides map[string]string) *PaddleOCRVLReader {
return &PaddleOCRVLReader{
timeout: paddleOCRVLRequestTimeout(),
timeout: requestTimeoutFromEnv("WEKNORA_PADDLEOCR_VL_TIMEOUT", defaultPaddleOCRVLTimeout),
endpoint: strings.TrimRight(overrides["paddleocr_vl_endpoint"], "/"),
useSeal: parseBoolOr(overrides["paddleocr_vl_use_seal_recognition"], true),
useChart: parseBoolOr(overrides["paddleocr_vl_use_chart_recognition"], false),
}
}
// paddleOCRVLRequestTimeout bounds a self-hosted layout-parsing request.
// The caller's context may impose an earlier deadline.
func paddleOCRVLRequestTimeout() time.Duration {
value := strings.TrimSpace(os.Getenv("WEKNORA_PADDLEOCR_VL_TIMEOUT"))
if value == "" {
return defaultPaddleOCRVLTimeout
}
timeout, err := time.ParseDuration(value)
if err != nil || timeout <= 0 {
logger.Warnf(context.Background(), "Invalid WEKNORA_PADDLEOCR_VL_TIMEOUT %q; using %s",
value, defaultPaddleOCRVLTimeout)
return defaultPaddleOCRVLTimeout
}
return timeout
}
func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
if c.endpoint == "" {
return &types.ReadResult{Error: "PaddleOCR-VL endpoint is not configured"}, nil

View File

@ -294,6 +294,8 @@ AWS S3 的 `S3_ACCESS_KEY` / `S3_SECRET_KEY` 可以**同时留空**,此时走
| `WEKNORA_HOUSEKEEPING_ENABLED` | 启用 | 回收卡在 processing 的脏数据 |
| `WEKNORA_DOCUMENT_PROCESS_TIMEOUT` / `WEKNORA_DOCREADER_CALL_TIMEOUT` | 2h / 30m | 文档处理任务与单次 RPC 超时 |
| `WEKNORA_PADDLEOCR_VL_TIMEOUT` | 1000s | 自建 PaddleOCR-VL HTTP 请求超时,支持正数 Go duration(如 `5400s`、`90m`);空值、无效值或非正数使用默认值。外层超时需留余量,例如本项 `90m`、DocReader `100m`、文档任务 `2h` |
| `WEKNORA_MINERU_TIMEOUT` | 1000s | 自建 MinerU 单次解析超时(V1 API 为整个解析任务,旧版为 `/file_parse` 请求),格式与默认值规则同上;超大 PDF 同样需要外层超时留余量 |
| `WEKNORA_MINERU_CLOUD_TIMEOUT` | 600s | MinerU 云端(mineru.net)轮询解析结果的最长时间,格式与默认值规则同上 |
沙箱后端、网络策略、脚本开关与个人环境变量使用空间配置/API 管理,见[技能与沙箱](../03-features/22-skills-sandbox.md)。长期记忆与自动标签均默认关闭,分别使用租户 memory_config 和知识库 auto_tag_config,不用全局环境变量替代各空间配置。

View File

@ -392,8 +392,10 @@ gRPC 响应中不再返回 chunks(`ReadResponse` 没有 chunk 字段);`Exc
| `WEKNORA_DOCUMENT_PROCESS_TIMEOUT` | Go duration | `2h` | 单个文档处理任务的总超时 |
| `WEKNORA_DOCREADER_CALL_TIMEOUT` | Go duration | `30m` | 单次 docreader 调用超时,须小于文档处理超时 |
| `WEKNORA_PADDLEOCR_VL_TIMEOUT` | Go duration | `1000s` | 自建 PaddleOCR-VL 引擎的 HTTP 请求超时;空值、无效值或非正数使用默认值 |
| `WEKNORA_MINERU_TIMEOUT` | Go duration | `1000s` | 自建 MinerU 引擎的单次解析超时(V1 API 覆盖上传到下载的整个任务,旧版覆盖 `/file_parse` 请求);空值、无效值或非正数使用默认值 |
| `WEKNORA_MINERU_CLOUD_TIMEOUT` | Go duration | `600s` | MinerU 云端(mineru.net)轮询解析结果的最长时间;空值、无效值或非正数使用默认值 |
处理大文件时从内到外逐级留出余量,例如 PaddleOCR-VL `90m`、docreader `100m`、文档处理 `2h`。
处理大文件时从内到外逐级留出余量,例如 PaddleOCR-VL 或 MinerU `90m`、docreader `100m`、文档处理 `2h`。
#### 安全与其他 {#_6-3-安全与其他}
@ -501,7 +503,7 @@ V1 流程的几个细节:
- 上传时附带 `sha256sum`,服务端已有相同文件时直接复用,不再传字节。
- 只有 `upload_url` 与 `mineru_endpoint` 同源(scheme、host、端口都相同)时才附带 API Key;跨源地址(如官方 API 下发的预签名对象存储 URL)不带 Key,并照常经过 SSRF 校验。
- 轮询从 2 秒开始指数退避,最长 30 秒一次;总时长与旧版一样是 1000 秒。超时或调用方取消时,会发 `DELETE /v1/parse/jobs/{id}` 取消服务端任务。
- 轮询从 2 秒开始指数退避,最长 30 秒一次;总时长与旧版一样默认 1000 秒,可用 `WEKNORA_MINERU_TIMEOUT` 调整。超时或调用方取消时,会发 `DELETE /v1/parse/jobs/{id}` 取消服务端任务。
- MinerU V1 服务的上传、任务状态都存在进程内存里,服务重启后正在轮询的任务会返回 404,本次解析直接失败。
- 「测试连接」在 V1 服务上额外请求一次需要鉴权的 `GET /v1/parse/jobs?limit=1`,用来发现 API Key 缺失或错误(`/v1/health` 本身不校验 Key)。