feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
main
wizardchen 2026-02-28 13:22:07 +08:00 committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions

View File

@ -101,12 +101,7 @@ JWT_SECRET=weknora-jwt-secret
# Embedding并发数,出现429错误时,可调小此参数
CONCURRENCY_POOL_SIZE=5
# Docreader 并发任务数(图片OCR/Caption等异步任务),默认 1
# 默认使用的 paddleocr 在高并发场景下,会出现异常,请谨慎设置
# IMAGE_MAX_CONCURRENT=1
# Docreader OCR后端(no_ocr, paddle, nanonets)
# OCR_BACKEND=paddle
# (Removed: IMAGE_MAX_CONCURRENT, OCR_BACKEND — moved to Go App module after lightweight refactoring)
# 如果使用ElasticSearch作为向量存储,需要配置以下参数
# ElasticSearch地址,例如 http://localhost:9200
@ -242,3 +237,9 @@ APK_MIRROR_ARG=mirrors.tencent.com
# Milvus 数据库名称(可选)
# MILVUS_DB_NAME=your_milvus_db_name
# Docreader 地址
DOCREADER_ADDR=docreader:50051
# Docreader 连接方式
DOCREADER_TRANSPORT=grpc

45
.env.lite.example 100644
View File

@ -0,0 +1,45 @@
# WeKnora Lite 配置模板
# 复制此文件为 .env.lite 并按需修改
# cp .env.lite.example .env.lite
GIN_MODE=release
# === 数据库 ===
DB_DRIVER=sqlite
DB_PATH=./data/weknora.db
# === 检索引擎(FTS5 + sqlite-vec)===
RETRIEVE_DRIVER=sqlite
# === 文件存储 ===
STORAGE_TYPE=local
LOCAL_STORAGE_BASE_DIR=./data/files
# === 流管理(内存,无 Redis)===
STREAM_MANAGER_TYPE=memory
# === LLM 服务 ===
# Ollama 本地服务(默认地址,按需修改)
OLLAMA_BASE_URL=http://127.0.0.1:11434
# 如使用其他 OpenAI 兼容服务,取消注释:
# OPENAI_API_KEY=sk-xxx
# OPENAI_BASE_URL=https://api.openai.com/v1
# === 安全配置(生产环境请务必修改!)===
TENANT_AES_KEY=CHANGE-ME-32-char-secret-key!!!!
JWT_SECRET=CHANGE-ME-jwt-secret
# === 功能开关 ===
NEO4J_ENABLE=false
WEKNORA_SANDBOX_MODE=disabled
ENABLE_GRAPH_RAG=false
DISABLE_REGISTRATION=false
# === 性能 ===
CONCURRENCY_POOL_SIZE=3
# Docreader 地址
DOCREADER_ADDR=127.0.0.1:50051
# Docreader 传输方式
DOCREADER_TRANSPORT=grpc

View File

@ -137,13 +137,13 @@ jobs:
platform: linux/arm64
runs: ubuntu-24.04-arm
runs-on: ${{ matrix.runs }}
timeout-minutes: 45
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
id: setup-buildx
- name: Login to Docker Hub
uses: docker/login-action@v3
@ -172,23 +172,6 @@ jobs:
# 显示版本信息
./scripts/get_version.sh info
- name: Build Cache for Docker
uses: actions/cache@v4
id: cache
with:
path: go-pkg-mod
key: ${{ env.PLATFORM_PAIR }}-go-build-cache-${{ hashFiles('**/go.sum') }}
- name: Inject go-build-cache
uses: reproducible-containers/buildkit-cache-dance@v3
with:
builder: ${{ steps.setup-buildx.outputs.name }}
cache-map: |
{
"go-pkg-mod": "/go/pkg/mod"
}
skip-extraction: ${{ steps.cache.outputs.cache-hit }}
- name: Build app Image
id: build
uses: docker/build-push-action@v3

View File

@ -0,0 +1,246 @@
name: Release Lite Binaries
on:
push:
tags:
- "v*"
workflow_dispatch:
inputs:
tag:
description: "Release tag (e.g. v0.2.0)"
required: true
concurrency:
group: release-lite-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: write
env:
GO_VERSION: "1.24"
NODE_VERSION: "22"
jobs:
# ── 1. Build frontend once, share as artifact ──
build-frontend:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_VERSION }}
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Build frontend
working-directory: frontend
run: |
npm ci
npm run build
- uses: actions/upload-artifact@v4
with:
name: frontend-dist
path: frontend/dist/
retention-days: 1
# ── 2. Build Go binary per platform (native CGO) ──
build-binary:
needs: build-frontend
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
goos: linux
goarch: amd64
- os: ubuntu-24.04-arm
goos: linux
goarch: arm64
- os: macos-13
goos: darwin
goarch: amd64
- os: macos-14
goos: darwin
goarch: arm64
runs-on: ${{ matrix.os }}
env:
CGO_ENABLED: 1
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- uses: actions/download-artifact@v4
with:
name: frontend-dist
path: web/
- name: Resolve version
id: ver
run: |
if [ -n "${{ inputs.tag }}" ]; then
VERSION="${{ inputs.tag }}"
else
VERSION="${GITHUB_REF#refs/tags/}"
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
echo "Building ${VERSION} for ${{ matrix.goos }}/${{ matrix.goarch }}"
- name: Build
run: |
export EDITION=lite
eval "$(./scripts/get_version.sh env)"
LDFLAGS="-w -s $(./scripts/get_version.sh ldflags)"
export CGO_CFLAGS="-Wno-deprecated-declarations"
if [ "${{ matrix.goos }}" = "darwin" ]; then
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
fi
go build -tags "sqlite_fts5" -ldflags="${LDFLAGS}" \
-o WeKnora-lite ./cmd/server
- name: Package tarball
run: |
ARCHIVE="WeKnora-lite_${{ steps.ver.outputs.version }}_${{ matrix.goos }}_${{ matrix.goarch }}"
mkdir -p "${ARCHIVE}/web"
cp WeKnora-lite "${ARCHIVE}/"
cp -r web/* "${ARCHIVE}/web/"
cp .env.lite.example "${ARCHIVE}/"
cp docs/LITE.md "${ARCHIVE}/README.md"
if [ -d config ]; then
cp -r config "${ARCHIVE}/config"
fi
if [ -d migrations/sqlite ]; then
mkdir -p "${ARCHIVE}/migrations/sqlite"
cp -r migrations/sqlite/* "${ARCHIVE}/migrations/sqlite/"
fi
if [ -f deploy/weknora-lite.service ]; then
cp deploy/weknora-lite.service "${ARCHIVE}/"
fi
tar czf "${ARCHIVE}.tar.gz" "${ARCHIVE}"
shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256"
- uses: actions/upload-artifact@v4
with:
name: release-${{ matrix.goos }}-${{ matrix.goarch }}
path: |
WeKnora-lite_*.tar.gz
WeKnora-lite_*.tar.gz.sha256
retention-days: 3
# ── 3. Create GitHub Release ──
release:
needs: build-binary
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
pattern: release-*
merge-multiple: true
- name: Resolve version
id: ver
run: |
if [ -n "${{ inputs.tag }}" ]; then
VERSION="${{ inputs.tag }}"
else
VERSION="${GITHUB_REF#refs/tags/}"
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
- name: Generate release notes
id: notes
run: |
cat > notes.md << 'NOTES'
## WeKnora Lite ${{ steps.ver.outputs.version }}
单二进制部署,零外部依赖(无需 Docker / PostgreSQL / Redis)。
### 快速开始
```bash
# 1. 解压
tar xzf WeKnora-lite_${{ steps.ver.outputs.version }}_<os>_<arch>.tar.gz
cd WeKnora-lite_${{ steps.ver.outputs.version }}_<os>_<arch>
# 2. 配置
cp .env.lite.example .env.lite
# 编辑 .env.lite,至少确认 OLLAMA_BASE_URL 正确
# 3. 启动 Ollama(如尚未运行)
ollama serve &
ollama pull qwen2.5:7b
ollama pull nomic-embed-text
# 4. 运行
set -a && source .env.lite && set +a
./WeKnora-lite
# 访问 http://localhost:8080
```
### 平台支持
| 文件 | 平台 |
|------|------|
| `WeKnora-lite_*_linux_amd64.tar.gz` | Linux x86_64 |
| `WeKnora-lite_*_linux_arm64.tar.gz` | Linux ARM64 |
| `WeKnora-lite_*_darwin_amd64.tar.gz` | macOS Intel |
| `WeKnora-lite_*_darwin_arm64.tar.gz` | macOS Apple Silicon |
详细文档见 [LITE.md](docs/LITE.md)。
NOTES
- name: Create release
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release create "${{ steps.ver.outputs.version }}" \
--title "WeKnora Lite ${{ steps.ver.outputs.version }}" \
--notes-file notes.md \
WeKnora-lite_*.tar.gz \
WeKnora-lite_*.tar.gz.sha256
# ── 4. Update Homebrew Formula ──
update-homebrew:
needs: release
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: main
token: ${{ secrets.GITHUB_TOKEN }}
- name: Resolve version
id: ver
run: |
if [ -n "${{ inputs.tag }}" ]; then
VERSION="${{ inputs.tag }}"
else
VERSION="${GITHUB_REF#refs/tags/}"
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
- name: Wait for release assets to be available
run: sleep 15
- name: Update Formula
run: ./scripts/update-homebrew-formula.sh "${{ steps.ver.outputs.version }}"
- name: Commit and push
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add Formula/weknora-lite.rb
if git diff --cached --quiet; then
echo "No changes to Formula"
else
git commit -m "formula: update weknora-lite to ${{ steps.ver.outputs.version }}"
git push origin main
fi

7
.gitignore vendored
View File

@ -1,5 +1,6 @@
# 忽略.env文件和其他包含敏感信息的配置文件
.env
.env.lite
# 但不忽略示例文件
!.env.example
*.pem
@ -27,9 +28,15 @@ logs/
*.pid
WeKnora
WeKnora-lite
/models/
test/data/mswag.txt
data/files/
data/weknora.db
data/weknora.db-wal
data/weknora.db-shm
web/
.venv/
**/__pycache__

View File

@ -0,0 +1,118 @@
class WeknoraLite < Formula
desc "Knowledge base management system — single-binary Lite edition"
homepage "https://github.com/Tencent/WeKnora"
version "0.0.0"
license "Apache-2.0"
on_macos do
on_arm do
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_darwin_arm64.tar.gz"
sha256 "PLACEHOLDER"
end
on_intel do
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_darwin_amd64.tar.gz"
sha256 "PLACEHOLDER"
end
end
on_linux do
on_arm do
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_linux_arm64.tar.gz"
sha256 "PLACEHOLDER"
end
on_intel do
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_linux_amd64.tar.gz"
sha256 "PLACEHOLDER"
end
end
def install
libexec.install "WeKnora-lite"
pkgshare.install "web" if File.directory?("web")
pkgshare.install "config" if File.directory?("config")
pkgshare.install ".env.lite.example"
doc.install "README.md"
pkgshare.install "migrations" if File.directory?("migrations")
(bin/"weknora-lite").write <<~SH
#!/bin/bash
CONFIG_DIR="${WEKNORA_CONFIG_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/weknora}"
DATA_DIR="${WEKNORA_DATA_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/weknora}"
mkdir -p "$DATA_DIR/files" "$CONFIG_DIR/config" 2>/dev/null
if [ ! -f "$CONFIG_DIR/config/config.yaml" ]; then
cp -r "#{pkgshare}/config/" "$CONFIG_DIR/config/"
fi
if [ ! -d "$CONFIG_DIR/migrations" ] && [ -d "#{pkgshare}/migrations" ]; then
ln -sf "#{pkgshare}/migrations" "$CONFIG_DIR/migrations"
fi
if [ ! -f "$CONFIG_DIR/.env.lite" ]; then
cp "#{pkgshare}/.env.lite.example" "$CONFIG_DIR/.env.lite"
sed -i '' "s|DB_PATH=.*|DB_PATH=$DATA_DIR/weknora.db|" "$CONFIG_DIR/.env.lite"
sed -i '' "s|LOCAL_STORAGE_BASE_DIR=.*|LOCAL_STORAGE_BASE_DIR=$DATA_DIR/files|" "$CONFIG_DIR/.env.lite"
rm -f "$CONFIG_DIR/.env.lite-e"
echo ""
echo "已创建配置文件: $CONFIG_DIR/.env.lite"
echo "请根据需要编辑(如修改 LLM 地址、安全密钥等)。"
echo ""
fi
set -a
source "$CONFIG_DIR/.env.lite"
set +a
export DB_PATH="${DB_PATH:-$DATA_DIR/weknora.db}"
export LOCAL_STORAGE_BASE_DIR="${LOCAL_STORAGE_BASE_DIR:-$DATA_DIR/files}"
export WEKNORA_WEB_DIR="${WEKNORA_WEB_DIR:-#{pkgshare}/web}"
cd "$CONFIG_DIR"
exec "#{libexec}/WeKnora-lite" "$@"
SH
end
def post_install
(var/"weknora").mkpath
(var/"log").mkpath
end
service do
run [bin/"weknora-lite"]
keep_alive true
working_dir var/"weknora"
log_path var/"log/weknora-lite.log"
error_log_path var/"log/weknora-lite.log"
end
def caveats
<<~EOS
前台运行:
weknora-lite
后台服务(推荐):
brew services start weknora-lite # 启动并开机自启
brew services stop weknora-lite # 停止
brew services restart weknora-lite # 重启
brew services info weknora-lite # 查看状态
日志:
#{var}/log/weknora-lite.log
首次运行会自动创建配置文件:
~/.config/weknora/.env.lite
数据存储在:
~/.local/share/weknora/
如需修改配置(LLM 服务地址、安全密钥等):
$EDITOR ~/.config/weknora/.env.lite
brew services restart weknora-lite
EOS
end
test do
assert_predicate bin/"weknora-lite", :executable?
end
end

View File

@ -1,4 +1,4 @@
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite
# Show help
help:
@ -56,6 +56,11 @@ help:
@echo " dev-status 查看开发环境状态"
@echo " dev-app 启动后端应用(本地运行,需先运行 dev-start)"
@echo " dev-frontend 启动前端(本地运行,需先运行 dev-start)"
@echo ""
@echo "Lite 模式(零外部依赖):"
@echo " build-lite 构建 Lite 版本(先构建前端到 web/,再构建 Go;SKIP_FRONTEND=1 跳过前端)"
@echo " run-lite 构建并启动 Lite 版本"
@echo " package-lite 构建并打包 Lite 发行包(tarball)"
# Go related variables
BINARY_NAME=WeKnora
@ -223,11 +228,42 @@ deps:
build-prod:
VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo "$${VERSION:-unknown}"); \
COMMIT_ID=$${COMMIT_ID:-unknown}; \
CGO_ENABLED=1 \
CGO_CFLAGS="-Wno-deprecated-declarations" \
CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \
BUILD_TIME=$${BUILD_TIME:-unknown}; \
GO_VERSION=$${GO_VERSION:-unknown}; \
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=standard' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH)
# Build Lite version (single binary, SQLite + in-memory queue)
# 会先构建前端到 web/,再构建 Go 二进制;SKIP_FRONTEND=1 可跳过前端
build-lite:
@if [ -f frontend/package.json ] && [ "$${SKIP_FRONTEND:-}" != "1" ]; then \
echo ">> Building frontend for Lite..."; \
(cd frontend && npm ci --prefer-offline && npm run build) && \
rm -rf web && cp -r frontend/dist web; \
elif [ "$${SKIP_FRONTEND:-}" = "1" ]; then \
echo ">> Skipping frontend (SKIP_FRONTEND=1)"; \
else \
echo ">> No frontend/package.json, skipping frontend"; \
fi
EDITION=lite eval "$$(./scripts/get_version.sh env)"; \
LDFLAGS="$$(EDITION=lite ./scripts/get_version.sh ldflags)"; \
CGO_ENABLED=1 \
CGO_CFLAGS="-Wno-deprecated-declarations" \
CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \
go build -tags "sqlite_fts5" -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME)-lite $(MAIN_PATH)
# Run Lite version with .env.lite defaults
run-lite: build-lite
@if [ ! -f .env.lite ]; then echo "Error: .env.lite not found"; exit 1; fi
@set -a && . ./.env.lite && set +a && ./$(BINARY_NAME)-lite
# Package Lite version into distributable tarball
package-lite:
./scripts/package-lite.sh
download_spatial:
go run cmd/download/duckdb/duckdb.go

View File

@ -0,0 +1,23 @@
[Unit]
Description=WeKnora Lite - Knowledge Base Management System
After=network.target
[Service]
Type=simple
User=weknora
Group=weknora
WorkingDirectory=/opt/weknora
EnvironmentFile=/opt/weknora/.env.lite
ExecStart=/opt/weknora/WeKnora-lite
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
# Security hardening
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/opt/weknora/data
[Install]
WantedBy=multi-user.target

View File

@ -107,16 +107,19 @@ services:
restart: "no"
docreader:
build:
context: .
dockerfile: docker/Dockerfile.docreader
image: wechatopenai/weknora-docreader:latest
container_name: WeKnora-docreader-dev
ports:
- "${DOCREADER_PORT:-50051}:50051"
volumes:
- docreader-tmp-dev:/tmp/docreader
environment:
- MINIO_ENDPOINT=minio:9000
- STORAGE_TYPE=${STORAGE_TYPE:-}
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
interval: 30s
@ -165,4 +168,5 @@ volumes:
neo4j-data-dev:
jaeger_data_dev:
qdrant_data_dev:
docreader-tmp-dev:

View File

@ -37,6 +37,7 @@ services:
- "${APP_PORT:-8080}:8080"
volumes:
- data-files:/data/files
- docreader-tmp:/tmp/docreader:ro
# Optional: mount custom config file
# - ./config/config.yaml:/app/config/config.yaml
# Optional: mount custom skills directory (allows adding skills without rebuilding image)
@ -83,7 +84,8 @@ services:
- QDRANT_USE_TLS=${QDRANT_USE_TLS:-false}
- MILVUS_ADDRESS=milvus:19530
- MILVUS_COLLECTION=${MILVUS_COLLECTION:-weknora_embeddings}
- DOCREADER_ADDR=docreader:50051
- DOCREADER_ADDR=${DOCREADER_ADDR:-docreader:50051}
- DOCREADER_TRANSPORT=${DOCREADER_TRANSPORT:-grpc}
- STORAGE_TYPE=${STORAGE_TYPE:-}
- LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-}
- AUTO_RECOVER_DIRTY=${AUTO_RECOVER_DIRTY:-true}
@ -146,10 +148,10 @@ services:
container_name: WeKnora-docreader
ports:
- "${DOCREADER_PORT:-50051}:50051"
volumes:
- docreader-tmp:/tmp/docreader
environment:
- MINIO_ENDPOINT=minio:9000
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
@ -312,6 +314,7 @@ networks:
volumes:
postgres-data:
data-files:
docreader-tmp:
jaeger_data:
minio_data:
neo4j-data:

View File

@ -1,5 +1,5 @@
# =========================
# 构建阶段
# 构建阶段(轻量化:仅文档解析 + 图片提取,无 OCR/VLM)
# =========================
FROM python:3.10.18-bookworm AS builder
@ -15,7 +15,6 @@ RUN apt-get update && apt-get install -y \
python3-dev \
libjpeg-dev \
zlib1g-dev \
libpq-dev \
libffi-dev \
libgl1 \
libglib2.0-0 \
@ -25,11 +24,10 @@ RUN apt-get update && apt-get install -y \
unzip \
&& rm -rf /var/lib/apt/lists/*
# 检查是否存在本地protoc安装包,如果存在则离线安装,否则在线安装,其他安装包按需求添加
# 检查是否存在本地protoc安装包,如果存在则离线安装,否则在线安装
ARG TARGETARCH
COPY packages/ /app/packages/
RUN echo "检查本地protoc安装包..." && \
# 根据目标架构选择正确的protoc包名
case ${TARGETARCH} in \
"amd64") PROTOC_ARCH="x86_64" ;; \
"arm64") PROTOC_ARCH="aarch_64" ;; \
@ -39,41 +37,18 @@ RUN echo "检查本地protoc安装包..." && \
PROTOC_PACKAGE="protoc-3.19.4-linux-${PROTOC_ARCH}.zip" && \
if [ -f "/app/packages/${PROTOC_PACKAGE}" ]; then \
echo "发现本地protoc安装包,将进行离线安装"; \
# 离线安装:使用本地包(精确路径避免歧义)
cp /app/packages/${PROTOC_PACKAGE} /app/ && \
unzip -o /app/${PROTOC_PACKAGE} -d /usr/local && \
chmod +x /usr/local/bin/protoc && \
rm -f /app/${PROTOC_PACKAGE}; \
else \
echo "未发现本地protoc安装包,将进行在线安装"; \
# 在线安装:从网络下载
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/${PROTOC_PACKAGE} && \
unzip -o ${PROTOC_PACKAGE} -d /usr/local && \
chmod +x /usr/local/bin/protoc && \
rm -f ${PROTOC_PACKAGE}; \
fi
# 预下载 PP-OCRv4 模型
RUN mkdir -p /root/.paddleocr/whl/det/ch && \
mkdir -p /root/.paddleocr/whl/rec/ch && \
mkdir -p /root/.paddleocr/whl/cls/ch && \
# 下载检测模型
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_det_infer.tar \
-O /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
tar -xf /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar -C /root/.paddleocr/whl/det/ch/ && \
# 下载识别模型
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar \
-O /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
tar -xf /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar -C /root/.paddleocr/whl/rec/ch/ && \
# 下载文本方向分类模型(用于判断文本是否需要旋转)
wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar \
-O /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar && \
tar -xf /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar -C /root/.paddleocr/whl/cls/ && \
# 清理压缩包
rm -f /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
rm -f /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
rm -f /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar
# 复制依赖文件
COPY docreader/pyproject.toml docreader/uv.lock ./
RUN pip install uv --break-system-packages && \
@ -82,15 +57,13 @@ RUN pip install uv --break-system-packages && \
# 复制源代码和生成脚本
COPY docreader docreader
# 生成 protobuf 代码
# 生成 protobuf 代码(使用 venv 中的 grpc_tools)
ENV PATH="/app/.venv/bin:${PATH}"
RUN chmod +x docreader/scripts/generate_proto.sh && \
bash docreader/scripts/generate_proto.sh
# 确保模型目录存在
RUN ls -la /root/.paddleocr/whl/
# =========================
# 运行阶段
# 运行阶段(轻量化)
# =========================
FROM python:3.10.18-bookworm AS runner
@ -100,16 +73,14 @@ RUN sed -i 's@http://deb.debian.org@https://mirrors.tuna.tsinghua.edu.cn@g' /etc
WORKDIR /app
# 安装运行时依赖
# 安装运行时依赖(已移除 OCR/PaddleOCR 相关依赖)
RUN apt-get update && apt-get install -y \
libjpeg62-turbo \
libpq5 \
wget \
gnupg \
libgl1 \
libglib2.0-0 \
antiword \
vim \
tar \
dpkg \
libxinerama1 \
@ -126,7 +97,6 @@ RUN apt-get update && apt-get install -y \
# 安装 grpc_health_probe
ARG TARGETARCH
RUN GRPC_HEALTH_PROBE_VERSION=v0.4.24 && \
# 根据目标架构选择正确的二进制文件
case ${TARGETARCH} in \
"amd64") ARCH="amd64" ;; \
"arm64") ARCH="arm64" ;; \
@ -135,25 +105,24 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.24 && \
esac && \
wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${ARCH} && \
chmod +x /bin/grpc_health_probe
# 从构建阶段复制已安装的依赖和生成的代码
ENV VIRTUAL_ENV=/app/.venv
COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
COPY --from=builder /usr/local/bin /usr/local/bin
COPY --from=builder /root/.paddleocr /root/.paddleocr
# 安装 Playwright 浏览器
# 安装 Playwright 浏览器(网页解析)
RUN python -m playwright install webkit
RUN python -m playwright install-deps webkit
# COPY docreader/scripts/download_deps.py download_deps.py
# RUN python -m download_deps
COPY docreader/pyproject.toml docreader/uv.lock ./
COPY --from=builder /app/docreader docreader
# 创建共享临时图片目录
RUN mkdir -p /tmp/docreader
# 暴露 gRPC 端口
EXPOSE 50051

View File

@ -13,42 +13,35 @@ import (
"google.golang.org/grpc/resolver"
)
// getMaxMessageSize returns the maximum gRPC message size in bytes.
// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable.
func getMaxMessageSize() int {
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
return size * 1024 * 1024
}
}
return 50 * 1024 * 1024 // default 50MB
return 50 * 1024 * 1024
}
// Logger is the default logger used by the client
var Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
// ImageInfo 表示一个图片的信息
type ImageInfo struct {
URL string // 图片URL(COS)
Caption string // 图片描述
OCRText string // OCR提取的文本
OriginalURL string // 原始图片URL
Start int // 图片在文本中的开始位置
End int // 图片在文本中的结束位置
// ImageRefInfo represents an image reference from a converted document.
type ImageRefInfo struct {
Filename string
OriginalRef string
MimeType string
StorageKey string
}
// Client represents a DocReader service client
// Client represents a DocReader service client.
type Client struct {
conn *grpc.ClientConn
proto.DocReaderClient
debug bool
}
// NewClient creates a new DocReader client with the specified address
func NewClient(addr string) (*Client, error) {
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
// 设置消息大小限制 (configurable via GRPC_MAX_MESSAGE_SIZE_MB)
maxMsgSize := getMaxMessageSize()
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
@ -75,19 +68,15 @@ func NewClient(addr string) (*Client, error) {
}, nil
}
// Close closes the client connection
func (c *Client) Close() error {
Logger.Printf("INFO: Closing DocReader client connection")
return c.conn.Close()
}
// SetDebug enables or disables debug logging
func (c *Client) SetDebug(debug bool) {
c.debug = debug
Logger.Printf("INFO: Debug logging set to %v", debug)
}
// Log logs a message with the appropriate level
func (c *Client) Log(level string, format string, args ...interface{}) {
if level == "DEBUG" && !c.debug {
return
@ -95,28 +84,20 @@ func (c *Client) Log(level string, format string, args ...interface{}) {
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
}
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
if chunk == nil || len(chunk.Images) == 0 {
// GetImageRefsFromResponse extracts image references from a ReadResponse.
func GetImageRefsFromResponse(resp *proto.ReadResponse) []ImageRefInfo {
if resp == nil || len(resp.ImageRefs) == 0 {
return nil
}
images := make([]ImageInfo, 0, len(chunk.Images))
for _, img := range chunk.Images {
images = append(images, ImageInfo{
URL: img.Url,
Caption: img.Caption,
OCRText: img.OcrText,
OriginalURL: img.OriginalUrl,
Start: int(img.Start),
End: int(img.End),
refs := make([]ImageRefInfo, 0, len(resp.ImageRefs))
for _, ref := range resp.ImageRefs {
refs = append(refs, ImageRefInfo{
Filename: ref.Filename,
OriginalRef: ref.OriginalRef,
MimeType: ref.MimeType,
StorageKey: ref.StorageKey,
})
}
return images
}
// HasImagesInChunk 判断一个Chunk是否包含图片
func HasImagesInChunk(chunk *proto.Chunk) bool {
return chunk != nil && len(chunk.Images) > 0
return refs
}

View File

@ -11,144 +11,75 @@ import (
)
func init() {
// 配置测试日志
log.SetOutput(os.Stdout)
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
log.Println("INFO: Initializing DocReader client tests")
}
func TestReadFromURL(t *testing.T) {
log.Println("INFO: Starting TestReadFromURL")
// 创建测试客户端
log.Println("INFO: Creating test client")
func TestReadURL(t *testing.T) {
client, err := NewClient("localhost:50051")
if err != nil {
log.Printf("ERROR: Failed to create client: %v", err)
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// 启用调试日志
client.SetDebug(true)
// 测试 ReadFromURL 方法
log.Println("INFO: Sending ReadFromURL request to server")
startTime := time.Now()
resp, err := client.ReadFromURL(
resp, err := client.Read(
context.Background(),
&proto.ReadFromURLRequest{
&proto.ReadRequest{
Url: "https://example.com",
Title: "test",
ReadConfig: &proto.ReadConfig{
ChunkSize: 512,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
EnableMultimodal: true,
},
},
)
log.Printf("INFO: Read(URL) completed in %v", time.Since(startTime))
requestDuration := time.Since(startTime)
if err != nil {
log.Printf("ERROR: ReadFromURL failed: %v", err)
t.Fatalf("ReadFromURL failed: %v", err)
t.Fatalf("Read failed: %v", err)
}
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
// 验证结果
chunkCount := len(resp.Chunks)
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
if chunkCount == 0 {
log.Println("WARN: Expected non-empty content but received none")
t.Error("Expected non-empty content")
if resp.Error != "" {
t.Fatalf("Read returned error: %s", resp.Error)
}
// 打印结果
for i, chunk := range resp.Chunks {
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
} else if i == 2 && chunkCount > 4 {
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
}
if resp.MarkdownContent == "" {
t.Error("Expected non-empty markdown content")
}
log.Println("INFO: TestReadFromURL completed successfully")
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(resp.ImageRefs))
}
func TestReadFromFileWithChunking(t *testing.T) {
log.Println("INFO: Starting TestReadFromFileWithChunking")
// 创建测试客户端
log.Println("INFO: Creating test client")
func TestReadFile(t *testing.T) {
client, err := NewClient("localhost:50051")
if err != nil {
log.Printf("ERROR: Failed to create client: %v", err)
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// 启用调试日志
client.SetDebug(true)
// 读取测试文件
log.Println("INFO: Reading test file")
fileContent, err := os.ReadFile("../testdata/test.md")
if err != nil {
log.Printf("ERROR: Failed to read test file: %v", err)
t.Fatalf("Failed to read test file: %v", err)
}
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
// 测试 ReadFromFile 方法,带分块参数
log.Println("INFO: Sending ReadFromFile request to server")
startTime := time.Now()
resp, err := client.ReadFromFile(
resp, err := client.Read(
context.Background(),
&proto.ReadFromFileRequest{
&proto.ReadRequest{
FileContent: fileContent,
FileName: "test.md",
FileType: "md",
ReadConfig: &proto.ReadConfig{
ChunkSize: 200,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
EnableMultimodal: true,
},
},
)
log.Printf("INFO: Read(file) completed in %v", time.Since(startTime))
requestDuration := time.Since(startTime)
if err != nil {
log.Printf("ERROR: ReadFromFile failed: %v", err)
t.Fatalf("ReadFromFile failed: %v", err)
t.Fatalf("Read failed: %v", err)
}
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
// 验证结果
chunkCount := len(resp.Chunks)
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
if chunkCount == 0 {
log.Println("WARN: Expected non-empty content but received none")
t.Error("Expected non-empty content")
if resp.Error != "" {
t.Fatalf("Read returned error: %s", resp.Error)
}
if resp.MarkdownContent == "" {
t.Error("Expected non-empty markdown content")
}
// 打印结果
for i, chunk := range resp.Chunks {
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
} else if i == 2 && chunkCount > 4 {
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
}
}
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
}
// 截断字符串以供日志打印
func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
imageRefs := GetImageRefsFromResponse(resp)
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(imageRefs))
}

View File

@ -52,58 +52,17 @@ class DocReaderConfig:
grpc_max_file_size_mb: int
grpc_port: int
# Image processing
image_max_concurrent: int
# Proxy
external_http_proxy: str
external_https_proxy: str
# OCR
ocr_backend: str
ocr_api_base_url: str
ocr_api_key: str
ocr_model: str
# VLM Caption
vlm_model_base_url: str
vlm_model_name: str
vlm_model_api_key: str
vlm_interface_type: str
# Storage
storage_type: str
cos_secret_id: str
cos_secret_key: str
cos_region: str
cos_bucket_name: str
cos_app_id: str
cos_path_prefix: str
cos_enable_old_domain: bool
minio_access_key_id: str
minio_secret_access_key: str
minio_bucket_name: str
minio_path_prefix: str
minio_endpoint: str
minio_public_endpoint: str
minio_use_ssl: bool
local_storage_base_dir: str
# Other
mineru_endpoint: str
# Temp image output directory (shared with Go app via volume, local mode fallback)
image_output_dir: str
def load_config() -> DocReaderConfig:
"""Load config from environment variables.
"""Load config from environment variables (lightweight version)."""
Naming convention (new): DOCREADER_*
Backward compatible keys are supported.
"""
# gRPC
grpc_max_workers = _get_int(["DOCREADER_GRPC_MAX_WORKERS", "GRPC_MAX_WORKERS"], 4)
grpc_max_file_size_mb = (
_get_int(["DOCREADER_GRPC_MAX_FILE_SIZE_MB", "MAX_FILE_SIZE_MB"], 50)
@ -112,12 +71,6 @@ def load_config() -> DocReaderConfig:
)
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
# Image processing
image_max_concurrent = _get_int(
["DOCREADER_IMAGE_MAX_CONCURRENT", "IMAGE_MAX_CONCURRENT"], 1
)
# Proxies
external_http_proxy = _get_str(
["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], ""
)
@ -125,95 +78,17 @@ def load_config() -> DocReaderConfig:
["DOCREADER_EXTERNAL_HTTPS_PROXY", "EXTERNAL_HTTPS_PROXY"], ""
)
# OCR
ocr_backend = _get_str(["DOCREADER_OCR_BACKEND", "OCR_BACKEND"], "")
ocr_api_base_url = _get_str(["DOCREADER_OCR_API_BASE_URL", "OCR_API_BASE_URL"], "")
ocr_api_key = _get_str(["DOCREADER_OCR_API_KEY", "OCR_API_KEY"], "")
ocr_model = _get_str(["DOCREADER_OCR_MODEL", "OCR_MODEL"], "")
# VLM Caption
vlm_model_base_url = _get_str(
["DOCREADER_VLM_MODEL_BASE_URL", "VLM_MODEL_BASE_URL"], ""
image_output_dir = _get_str(
["DOCREADER_IMAGE_OUTPUT_DIR", "IMAGE_OUTPUT_DIR"], "/tmp/docreader"
)
vlm_model_name = _get_str(["DOCREADER_VLM_MODEL_NAME", "VLM_MODEL_NAME"], "")
vlm_model_api_key = _get_str(
["DOCREADER_VLM_MODEL_API_KEY", "VLM_MODEL_API_KEY"], ""
)
vlm_interface_type = _get_str(
["DOCREADER_VLM_INTERFACE_TYPE", "VLM_INTERFACE_TYPE"], "openai"
).lower()
# Storage
storage_type = _get_str(["DOCREADER_STORAGE_TYPE", "STORAGE_TYPE"], "cos").lower()
# COS
cos_secret_id = _get_str(["DOCREADER_COS_SECRET_ID", "COS_SECRET_ID"], "")
cos_secret_key = _get_str(["DOCREADER_COS_SECRET_KEY", "COS_SECRET_KEY"], "")
cos_region = _get_str(["DOCREADER_COS_REGION", "COS_REGION"], "")
cos_bucket_name = _get_str(["DOCREADER_COS_BUCKET_NAME", "COS_BUCKET_NAME"], "")
cos_app_id = _get_str(["DOCREADER_COS_APP_ID", "COS_APP_ID"], "")
cos_path_prefix = _get_str(["DOCREADER_COS_PATH_PREFIX", "COS_PATH_PREFIX"], "")
cos_enable_old_domain = _get_bool(
["DOCREADER_COS_ENABLE_OLD_DOMAIN", "COS_ENABLE_OLD_DOMAIN"], True
)
# MinIO
minio_access_key_id = _get_str(
["DOCREADER_MINIO_ACCESS_KEY_ID", "MINIO_ACCESS_KEY_ID"], "minioadmin"
)
minio_secret_access_key = _get_str(
["DOCREADER_MINIO_SECRET_ACCESS_KEY", "MINIO_SECRET_ACCESS_KEY"], "minioadmin"
)
minio_bucket_name = _get_str(
["DOCREADER_MINIO_BUCKET_NAME", "MINIO_BUCKET_NAME"], "WeKnora"
)
minio_path_prefix = _get_str(
["DOCREADER_MINIO_PATH_PREFIX", "MINIO_PATH_PREFIX"], ""
)
minio_endpoint = _get_str(["DOCREADER_MINIO_ENDPOINT", "MINIO_ENDPOINT"], "")
minio_public_endpoint = _get_str(
["DOCREADER_MINIO_PUBLIC_ENDPOINT", "MINIO_PUBLIC_ENDPOINT"], ""
)
minio_use_ssl = _get_bool(["DOCREADER_MINIO_USE_SSL", "MINIO_USE_SSL"], False)
# Local storage
local_storage_base_dir = "./data/files"
# Other
mineru_endpoint = _get_str(["DOCREADER_MINERU_ENDPOINT", "MINERU_ENDPOINT"], "")
return DocReaderConfig(
grpc_max_workers=grpc_max_workers,
grpc_max_file_size_mb=grpc_max_file_size_mb,
grpc_port=grpc_port,
image_max_concurrent=image_max_concurrent,
external_http_proxy=external_http_proxy,
external_https_proxy=external_https_proxy,
ocr_backend=ocr_backend,
ocr_api_base_url=ocr_api_base_url,
ocr_api_key=ocr_api_key,
ocr_model=ocr_model,
vlm_model_base_url=vlm_model_base_url,
vlm_model_name=vlm_model_name,
vlm_model_api_key=vlm_model_api_key,
vlm_interface_type=vlm_interface_type,
storage_type=storage_type,
cos_secret_id=cos_secret_id,
cos_secret_key=cos_secret_key,
cos_region=cos_region,
cos_bucket_name=cos_bucket_name,
cos_app_id=cos_app_id,
cos_path_prefix=cos_path_prefix,
cos_enable_old_domain=cos_enable_old_domain,
minio_access_key_id=minio_access_key_id,
minio_secret_access_key=minio_secret_access_key,
minio_bucket_name=minio_bucket_name,
minio_path_prefix=minio_path_prefix,
minio_endpoint=minio_endpoint,
minio_public_endpoint=minio_public_endpoint,
minio_use_ssl=minio_use_ssl,
local_storage_base_dir=local_storage_base_dir,
mineru_endpoint=mineru_endpoint,
image_output_dir=image_output_dir,
)
@ -223,56 +98,12 @@ CONFIG = load_config()
def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
cfg = CONFIG
d: Dict[str, Any] = {
# gRPC
"DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers,
"DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb,
"DOCREADER_GRPC_PORT": cfg.grpc_port,
# Image processing
"DOCREADER_IMAGE_MAX_CONCURRENT": cfg.image_max_concurrent,
# Proxy
"DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy,
"DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy,
# OCR
"DOCREADER_OCR_BACKEND": cfg.ocr_backend,
"DOCREADER_OCR_API_BASE_URL": cfg.ocr_api_base_url,
"DOCREADER_OCR_API_KEY": _mask_secret(cfg.ocr_api_key)
if mask_secrets
else cfg.ocr_api_key,
"DOCREADER_OCR_MODEL": cfg.ocr_model,
# VLM
"DOCREADER_VLM_MODEL_BASE_URL": cfg.vlm_model_base_url,
"DOCREADER_VLM_MODEL_NAME": cfg.vlm_model_name,
"DOCREADER_VLM_MODEL_API_KEY": _mask_secret(cfg.vlm_model_api_key)
if mask_secrets
else cfg.vlm_model_api_key,
"DOCREADER_VLM_INTERFACE_TYPE": cfg.vlm_interface_type,
# Storage
"DOCREADER_STORAGE_TYPE": cfg.storage_type,
"DOCREADER_COS_SECRET_ID": _mask_secret(cfg.cos_secret_id)
if mask_secrets
else cfg.cos_secret_id,
"DOCREADER_COS_SECRET_KEY": _mask_secret(cfg.cos_secret_key)
if mask_secrets
else cfg.cos_secret_key,
"DOCREADER_COS_REGION": cfg.cos_region,
"DOCREADER_COS_BUCKET_NAME": cfg.cos_bucket_name,
"DOCREADER_COS_APP_ID": cfg.cos_app_id,
"DOCREADER_COS_PATH_PREFIX": cfg.cos_path_prefix,
"DOCREADER_COS_ENABLE_OLD_DOMAIN": cfg.cos_enable_old_domain,
"DOCREADER_MINIO_ACCESS_KEY_ID": _mask_secret(cfg.minio_access_key_id)
if mask_secrets
else cfg.minio_access_key_id,
"DOCREADER_MINIO_SECRET_ACCESS_KEY": _mask_secret(cfg.minio_secret_access_key)
if mask_secrets
else cfg.minio_secret_access_key,
"DOCREADER_MINIO_BUCKET_NAME": cfg.minio_bucket_name,
"DOCREADER_MINIO_PATH_PREFIX": cfg.minio_path_prefix,
"DOCREADER_MINIO_ENDPOINT": cfg.minio_endpoint,
"DOCREADER_MINIO_PUBLIC_ENDPOINT": cfg.minio_public_endpoint,
"DOCREADER_MINIO_USE_SSL": cfg.minio_use_ssl,
"DOCREADER_LOCAL_STORAGE_BASE_DIR": cfg.local_storage_base_dir,
# Other
"DOCREADER_MINERU_ENDPOINT": cfg.mineru_endpoint,
"DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir,
}
return d

View File

@ -13,118 +13,130 @@ from grpc_health.v1.health import HealthServicer
from docreader import config
from docreader.config import CONFIG
from docreader.models.read_config import ChunkingConfig
from docreader.parser import Parser
from docreader.proto import docreader_pb2_grpc
from docreader.parser.registry import registry
from docreader.proto.docreader_pb2 import (
Chunk,
Image,
ReadConfig,
ReadFromFileRequest,
ReadFromURLRequest,
ReadRequest,
ReadResponse,
StorageProvider,
ImageRef,
ListEnginesResponse,
ParserEngineInfo,
)
from docreader.utils.request import init_logging_request_id, request_id_context
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
# cannot be encoded to UTF-8
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
def to_valid_utf8_text(s: Optional[str]) -> str:
"""Return a UTF-8 safe string for protobuf.
- Replace any surrogate code points with U+FFFD
- Re-encode with errors='replace' to ensure valid UTF-8
"""
if not s:
return ""
s = _SURROGATE_RE.sub("\ufffd", s)
return s.encode("utf-8", errors="replace").decode("utf-8")
# Ensure no existing handlers
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
# Configure logging - use stdout
handler = logging.StreamHandler(sys.stdout)
logging.root.addHandler(handler)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.info("Initializing server logging")
_level_name = (os.environ.get("LOG_LEVEL") or "INFO").upper()
_level = getattr(logging, _level_name, logging.INFO)
logging.root.setLevel(_level)
logger = logging.getLogger(__name__)
logger.info("Initializing server logging, level=%s", _level_name)
# Initialize request ID logging
init_logging_request_id()
parser = Parser()
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
``images`` is a dict of {relative_path: raw_data} where raw_data is
base64-encoded string or raw bytes.
def create_chunking_config(read_config: ReadConfig):
"""Create ChunkingConfig from ReadConfig request.
``storage_map`` is a dict from the request's config.image_storage map.
When it contains a valid "provider" key, images are uploaded to shared object
storage and storage_key is set (no inline bytes, saving gRPC message size).
Args:
read_config: The read_config from the gRPC request
Returns:
ChunkingConfig: Configured chunking configuration object
Returns (image_dir_path, list[ImageRef]).
"""
# Extract chunking parameters
chunk_size = read_config.chunk_size or 512
chunk_overlap = read_config.chunk_overlap or 50
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
separators = (
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
)
enable_multimodal = read_config.enable_multimodal or False
import base64
logger.info(
f"Using chunking config: size={chunk_size}, "
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
)
if not images:
return "", []
# Extract storage config
sc = read_config.storage_config
storage_config = {
"provider": StorageProvider.Name(sc.provider),
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
"secret_access_key": sc.secret_access_key,
"app_id": sc.app_id,
"path_prefix": sc.path_prefix,
}
logger.info(
f"Using Storage config: provider={storage_config.get('provider')}, "
f"bucket={storage_config['bucket_name']}"
)
# Extract VLM config
vlm_config = {
"model_name": read_config.vlm_config.model_name,
"base_url": read_config.vlm_config.base_url,
"api_key": read_config.vlm_config.api_key,
"interface_type": read_config.vlm_config.interface_type or "openai",
mime_map = {
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}"
)
storage_client = None
use_storage = False
provider = (storage_map or {}).get("provider", "")
if provider:
try:
from docreader.parser.storage import create_storage
storage_client = create_storage(storage_map)
use_storage = True
logger.info("Using shared storage (%s) for image upload", provider)
except Exception as e:
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
# Create and return ChunkingConfig
return ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
storage_config=storage_config,
vlm_config=vlm_config,
)
base_dir = CONFIG.image_output_dir
output_dir = os.path.join(base_dir, request_id, "images")
wrote_to_dir = False
if not use_storage:
try:
os.makedirs(output_dir, exist_ok=True)
wrote_to_dir = True
except OSError:
logger.warning("Cannot write to image output dir %s", output_dir)
refs = []
for ref_path, b64data in images.items():
try:
img_bytes = base64.b64decode(b64data)
except Exception:
img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data
fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png"
ext = os.path.splitext(fname)[1].lower()
mime = mime_map.get(ext, "application/octet-stream")
if use_storage and storage_client:
storage_key = storage_client.upload_bytes(img_bytes, ext)
if storage_key:
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
storage_key=storage_key,
))
continue
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
if wrote_to_dir:
dest = os.path.join(output_dir, fname)
try:
with open(dest, "wb") as f:
f.write(img_bytes)
except OSError as e:
logger.warning("Failed to write image %s: %s", dest, e)
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
image_data=img_bytes,
))
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
return image_dir, refs
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
@ -132,152 +144,90 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
super().__init__()
self.parser = Parser()
def ReadFromFile(self, request: ReadFromFileRequest, context):
# Get or generate request ID
request_id = (
request.request_id
if hasattr(request, "request_id") and request.request_id
else str(uuid.uuid4())
)
def Read(self, request: ReadRequest, context):
"""Unified read: file mode (file_content set) or URL mode (url set)."""
request_id = request.request_id or str(uuid.uuid4())
is_url = bool(request.url)
# Use request ID context
with request_id_context(request_id):
try:
# Get file type
file_type = (
request.file_type or os.path.splitext(request.file_name)[1][1:]
)
logger.info(
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
)
logger.info(f"File content size: {len(request.file_content)} bytes")
cfg = request.config
parser_engine = cfg.parser_engine if cfg else ""
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
# Create chunking config
chunking_config = create_chunking_config(request.read_config)
if is_url:
logger.info("Read(URL): url=%s", request.url)
result = self.parser.parse_url(
request.url,
request.title,
parser_engine=parser_engine,
engine_overrides=engine_overrides,
)
source_desc = request.url
else:
file_type = (
request.file_type or os.path.splitext(request.file_name)[1][1:]
)
logger.info(
"Read(File): file=%s, type=%s, size=%d bytes",
request.file_name, file_type, len(request.file_content),
)
result = self.parser.parse_file(
request.file_name,
file_type,
request.file_content,
parser_engine=parser_engine,
engine_overrides=engine_overrides,
)
source_desc = request.file_name
# Parse file
logger.info("Starting file parsing process")
result = self.parser.parse_file(
request.file_name, file_type, request.file_content, chunking_config
)
if not result:
error_msg = "Failed to parse file"
if not result or not result.content:
error_msg = f"Failed to parse: {source_desc}"
logger.error(error_msg)
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(error_msg)
return ReadResponse()
# Convert to protobuf message
logger.info(
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
)
# Build response, including image info
response = ReadResponse(
chunks=[
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
]
)
logger.info(f"Response size: {response.ByteSize()} bytes")
return response
except Exception as e:
error_msg = f"Error reading file: {str(e)}"
logger.error(error_msg)
logger.info(f"Detailed traceback: {traceback.format_exc()}")
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
return ReadResponse(error=str(e))
def ReadFromURL(self, request: ReadFromURLRequest, context):
# Get or generate request ID
request_id = (
request.request_id
if hasattr(request, "request_id") and request.request_id
else str(uuid.uuid4())
)
# Use request ID context
with request_id_context(request_id):
try:
logger.info(f"Received ReadFromURL request for URL: {request.url}")
# Create chunking config
chunking_config = create_chunking_config(request.read_config)
# Parse URL
logger.info("Starting URL parsing process")
result = self.parser.parse_url(
request.url, request.title, chunking_config
)
if not result:
error_msg = "Failed to parse URL"
logger.error(error_msg)
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(error_msg)
return ReadResponse(error=error_msg)
# Convert to protobuf message, including image info
logger.info(
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
_c = to_valid_utf8_text
image_dir, image_refs = _resolve_images(
result.images, request_id, storage_map=storage_map
)
response = ReadResponse(
chunks=[
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
]
markdown_content=_c(result.content),
image_refs=image_refs,
image_dir_path=image_dir,
)
logger.info(
"Read response: content_len=%d, images=%d",
len(result.content), len(image_refs),
)
logger.info(f"Response size: {response.ByteSize()} bytes")
return response
except Exception as e:
error_msg = f"Error reading URL: {str(e)}"
error_msg = f"Error reading document: {e}"
logger.error(error_msg)
logger.info(f"Detailed traceback: {traceback.format_exc()}")
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
logger.info("Traceback: %s", traceback.format_exc())
return ReadResponse(error=str(e))
def _convert_chunk_to_proto(self, chunk):
"""Convert internal Chunk object to protobuf Chunk message
Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates).
"""
# Clean helper for strings
_c = to_valid_utf8_text
proto_chunk = Chunk(
content=_c(getattr(chunk, "content", None)),
seq=getattr(chunk, "seq", 0),
start=getattr(chunk, "start", 0),
end=getattr(chunk, "end", 0),
)
# If chunk has images attribute and is not empty, add image info
if hasattr(chunk, "images") and chunk.images:
logger.info(
f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}"
def ListEngines(self, request, context):
overrides = dict(getattr(request, "config_overrides", None) or {})
engines_data = registry.list_engines(overrides=overrides or None)
engines = [
ParserEngineInfo(
name=e["name"],
description=e["description"],
file_types=e["file_types"],
available=e.get("available", True),
unavailable_reason=e.get("unavailable_reason", ""),
)
for img_info in chunk.images:
# img_info expected as dict
proto_image = Image(
url=_c(img_info.get("cos_url", "")),
caption=_c(img_info.get("caption", "")),
ocr_text=_c(img_info.get("ocr_text", "")),
original_url=_c(img_info.get("original_url", "")),
start=int(img_info.get("start", 0) or 0),
end=int(img_info.get("end", 0) or 0),
)
proto_chunk.images.append(proto_image)
return proto_chunk
for e in engines_data
]
return ListEnginesResponse(engines=engines)
def main():
# Print effective env/config at startup
config.print_config()
# Create server
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
options=[
@ -286,24 +236,18 @@ def main():
],
)
# Register services
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
# Register health check service
health_servicer = HealthServicer()
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
# Set listen address
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
# Start service
server.start()
logger.info(f"Server started on port {CONFIG.grpc_port}")
logger.info("Server started on port %d", CONFIG.grpc_port)
logger.info("Server is ready to accept connections")
try:
# Wait for service termination
server.wait_for_termination()
except KeyboardInterrupt:
logger.info("Received termination signal, shutting down server")

View File

@ -1,27 +1,17 @@
from dataclasses import dataclass, field
from dataclasses import dataclass
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""Legacy config kept for backward compatibility.
After the lightweight refactoring, chunking is done in Go.
This class is only kept so existing parser constructors don't break.
"""
# Maximum size of each chunk in tokens/chars
chunk_size: int = 512
# Number of tokens/chars to overlap between chunks
chunk_overlap: int = 50
# Text separators in order of priority
separators: list[str] = field(default_factory=lambda: ["\n\n", "\n", "。"])
# Whether to enable multimodal processing (text + images)
separators: list[str] | None = None
enable_multimodal: bool = False
# Preferred field name going forward
storage_config: dict[str, str] = field(default_factory=dict)
# VLM configuration for image captioning
vlm_config: dict[str, str] = field(default_factory=dict)
storage_config: dict[str, str] | None = None
vlm_config: dict[str, str] | None = None

View File

@ -13,7 +13,6 @@ The parsers extract content from documents and can split them into
meaningful chunks for further processing and indexing.
"""
from .csv_parser import CSVParser
from .doc_parser import DocParser
from .docx2_parser import Docx2Parser
from .excel_parser import ExcelParser
@ -21,19 +20,19 @@ from .image_parser import ImageParser
from .markdown_parser import MarkdownParser
from .parser import Parser
from .pdf_parser import PDFParser
from .text_parser import TextParser
from .registry import ParserEngineRegistry, registry
from .web_parser import WebParser
# Export public classes and modules
__all__ = [
"Docx2Parser", # Parser for .docx files (modern Word documents)
"DocParser", # Parser for .doc files (legacy Word documents)
"PDFParser", # Parser for PDF documents
"MarkdownParser", # Parser for Markdown text files
"TextParser", # Parser for plain text files
"ImageParser", # Parser for images with text content
"WebParser", # Parser for web pages
"Parser", # Main parser factory that selects the appropriate parser
"CSVParser", # Parser for CSV files
"ExcelParser", # Parser for Excel files
"Docx2Parser",
"DocParser",
"PDFParser",
"MarkdownParser",
"ImageParser",
"WebParser",
"Parser",
"ExcelParser",
"ParserEngineRegistry",
"registry",
]

File diff suppressed because it is too large Load Diff

View File

@ -1,387 +0,0 @@
import json
import logging
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Union
import ollama
import requests
from docreader.config import CONFIG
logger = logging.getLogger(__name__)
@dataclass
class ImageUrl:
"""Image URL data structure for caption requests."""
url: Optional[str] = None
detail: Optional[str] = None
@dataclass
class Content:
"""Content data structure that can contain text or image URL."""
type: Optional[str] = None
text: Optional[str] = None
image_url: Optional[ImageUrl] = None
@dataclass
class SystemMessage:
"""System message for VLM model requests."""
role: Optional[str] = None
content: Optional[str] = None
@dataclass
class UserMessage:
"""User message for VLM model requests, can contain multiple content items."""
role: Optional[str] = None
content: List[Content] = field(default_factory=list)
@dataclass
class CompletionRequest:
"""Request structure for VLM model completion API."""
model: str
temperature: float
top_p: float
messages: List[Union[SystemMessage, UserMessage]]
user: str
@dataclass
class Model:
"""Model identifier structure."""
id: str
@dataclass
class ModelsResp:
"""Response structure for available models API."""
data: List[Model] = field(default_factory=list)
@dataclass
class Message:
"""Message structure in API response."""
role: Optional[str] = None
content: Optional[str] = None
tool_calls: Optional[str] = None
@dataclass
class Choice:
"""Choice structure in API response."""
message: Optional[Message] = None
@dataclass
class Usage:
"""Token usage information in API response."""
prompt_tokens: Optional[int] = 0
total_tokens: Optional[int] = 0
completion_tokens: Optional[int] = 0
@dataclass
class CaptionChatResp:
"""Response structure for caption chat API."""
id: Optional[str] = None
created: Optional[int] = None
model: Optional[Model] = None
object: Optional[str] = None
choices: List[Choice] = field(default_factory=list)
usage: Optional[Usage] = None
@staticmethod
def from_json(json_data: dict) -> "CaptionChatResp":
"""
Parse API response JSON into a CaptionChatResp object.
Args:
json_data: The JSON response from the API
Returns:
A parsed CaptionChatResp object
"""
logger.info("Parsing CaptionChatResp from JSON")
# Manually parse nested fields with safe field extraction
choices = []
for choice in json_data.get("choices", []):
message_data = choice.get("message", {})
message = Message(
role=message_data.get("role"),
content=message_data.get("content"),
tool_calls=message_data.get("tool_calls"),
)
choices.append(Choice(message=message))
# Handle usage with safe field extraction
usage_data = json_data.get("usage", {})
usage = None
if usage_data:
usage = Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0),
total_tokens=usage_data.get("total_tokens", 0),
completion_tokens=usage_data.get("completion_tokens", 0),
)
logger.info(
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
)
return CaptionChatResp(
id=json_data.get("id"),
created=json_data.get("created"),
model=json_data.get("model"),
object=json_data.get("object"),
choices=choices,
usage=usage,
)
def choice_data(self) -> str:
"""
Extract the content from the first choice in the response.
Returns:
The content string from the first choice, or empty string if no choices
"""
if (
not self.choices
or not self.choices[0]
or not self.choices[0].message
or not self.choices[0].message.content
):
logger.warning("No choices available in response")
return ""
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
class Caption:
"""
Service for generating captions for images using a Vision Language Model.
Uses an external API to process images and return textual descriptions.
"""
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
"""
Initialize the Caption service with configuration
from parameters or environment variables.
"""
logger.info("Initializing Caption service")
# Default prompt for image captioning in Chinese: "Briefly describe the main content of the image"
self.prompt = """简单凝炼的描述图片的主要内容"""
# API request timeout in seconds
self.timeout = 30
# Use provided VLM config if available,
# otherwise fall back to environment variables
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
# Build completion URL from provided base URL
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
self.model = vlm_config.get("model_name", "")
self.api_key = vlm_config.get("api_key", "")
# Interface type: "ollama" or "openai" (default)
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
# Fall back to environment variables if config not provided
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
self.model = CONFIG.vlm_model_name
self.api_key = CONFIG.vlm_model_api_key
self.interface_type = CONFIG.vlm_interface_type
# Validate interface type - must be either "ollama" or "openai"
if self.interface_type not in ["ollama", "openai"]:
logger.warning(
f"Unknown interface type: {self.interface_type}, defaulting to openai"
)
self.interface_type = "openai"
logger.info(
f"Configured with model: {self.model}, "
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
)
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
"""
Call the Caption API to generate a description for the given image.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
CaptionChatResp object if successful, None otherwise
"""
logger.info("Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50]}...")
# Route to appropriate API based on interface type
if self.interface_type == "ollama":
return self._call_ollama_api(image_data)
else:
return self._call_openai_api(image_data)
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call Ollama API for image captioning using base64 encoded image data."""
# Extract host URL by removing the chat completions endpoint
# Handle both "/v1/chat/completions" and "/chat/completions" patterns
host = self.completion_url.replace("/v1/chat/completions", "").replace("/chat/completions", "")
# Initialize Ollama client with host and timeout
client = ollama.Client(
host=host,
timeout=self.timeout,
)
try:
logger.info(f"Calling Ollama API with model: {self.model}")
# Call Ollama API with base64 encoded image
# Prompt: "Briefly describe the main content of the image"
response = client.generate(
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # Pass base64 encoded image data
options={
"temperature": 0.1
}, # Low temperature for more deterministic output
stream=False,
)
# Construct response object in standard format
caption_resp = CaptionChatResp(
id="ollama_response",
created=int(time.time()),
model=Model(id=self.model),
object="chat.completion",
choices=[
Choice(message=Message(role="assistant", content=response.response))
],
)
logger.info("Successfully received response from Ollama API")
return caption_resp
except Exception as e:
logger.error(f"Error calling Ollama API: {e}")
return None
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call OpenAI-compatible API for image captioning."""
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
# Construct user message with text prompt and base64 encoded image
user_msg = UserMessage(
role="user",
content=[
Content(type="text", text=self.prompt),
Content(
type="image_url",
image_url=ImageUrl(
url="data:image/png;base64," + image_base64, detail="auto"
),
),
],
)
# Build completion request with model parameters
gpt_req = CompletionRequest(
model=self.model,
temperature=0.3, # Moderate randomness for balanced output
top_p=0.8, # Nucleus sampling parameter
messages=[user_msg],
user="abc",
)
# Set up HTTP headers for the API request
headers = {
"Content-Type": "application/json",
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
}
# Add authorization header if API key is provided
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
logger.info(
f"Sending request to OpenAI-compatible API with model: {self.model}"
)
# Send POST request to the API endpoint
response = requests.post(
self.completion_url,
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
headers=headers,
timeout=self.timeout,
)
# Check for successful response
if response.status_code != 200:
logger.error(
f"OpenAI API returned non-200 status code: {response.status_code}"
)
response.raise_for_status()
logger.info(f"Received from OpenAI with status: {response.status_code}")
logger.info("Converting response to CaptionChatResp object")
# Parse JSON response into structured object
caption_resp = CaptionChatResp.from_json(response.json())
if caption_resp.usage:
logger.info(
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
f"completion_tokens={caption_resp.usage.completion_tokens}"
)
return caption_resp
except requests.exceptions.Timeout:
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
return None
except requests.exceptions.RequestException as e:
logger.error(f"Request error calling OpenAI-compatible API: {e}")
return None
except Exception as e:
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
return None
def get_caption(self, image_data: str) -> str:
"""
Get a caption for the provided image data.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
Caption text as string, or empty string if captioning failed
"""
logger.info("Getting caption for image")
if not image_data or self.completion_url is None:
logger.error("Image data is not set")
return ""
caption_resp = self._call_caption_api(image_data)
if caption_resp:
caption = caption_resp.choice_data()
caption_length = len(caption)
logger.info(f"Successfully generated caption of length {caption_length}")
logger.info(
f"Caption: {caption[:50]}..."
if caption_length > 50
else f"Caption: {caption}"
)
return caption
logger.warning("Failed to get caption from Caption API")
return ""

View File

@ -1,101 +0,0 @@
"""
CSV Parser Module
This module provides a parser for CSV (Comma-Separated Values) files.
It converts CSV data into a Document with structured chunks, where each row
becomes a separate chunk with key-value pairs.
"""
import logging
from io import BytesIO
from typing import List
import pandas as pd
from docreader.models.document import Chunk, Document
from docreader.parser.base_parser import BaseParser
logger = logging.getLogger(__name__)
class CSVParser(BaseParser):
"""
Parser for CSV files that converts tabular data into structured text.
This parser reads CSV content and transforms each row into a formatted string
with column-value pairs. Each row is stored as a separate Chunk in the Document,
allowing for granular access to individual records.
The output format for each row is:
"column1: value1, column2: value2, column3: value3\n"
Usage:
parser = CSVParser()
with open("data.csv", "rb") as f:
document = parser.parse_into_text(f.read())
"""
def parse_into_text(self, content: bytes) -> Document:
"""Parse CSV content into a Document with structured chunks.
Each row in the CSV is converted into a formatted string and stored as
a separate Chunk. The chunks maintain sequential order and track their
position in the overall document.
Args:
content: Raw bytes content of the CSV file
Returns:
Document: A Document object containing:
- content: Full text with all rows concatenated
- chunks: List of Chunk objects, one per CSV row
Note:
Bad lines in the CSV are automatically skipped using pandas'
on_bad_lines="skip" parameter.
"""
chunks: List[Chunk] = []
text: List[str] = []
start, end = 0, 0
# Read CSV content into a pandas DataFrame, skipping malformed lines
df = pd.read_csv(BytesIO(content), on_bad_lines="skip")
# Process each row in the DataFrame
for i, (idx, row) in enumerate(df.iterrows()):
# Format row as "column: value" pairs separated by commas
content_row = (
",".join(
f"{col.strip()}: {str(row[col]).strip()}" for col in df.columns
)
+ "\n"
)
# Update end position for this chunk
end += len(content_row)
text.append(content_row)
# Create a chunk for this row with position tracking
chunks.append(Chunk(content=content_row, seq=i, start=start, end=end))
# Update start position for next chunk
start = end
return Document(
content="".join(text),
chunks=chunks,
)
if __name__ == "__main__":
# Example usage: Parse a CSV file and display its content
logging.basicConfig(level=logging.DEBUG)
your_file = "/path/to/your/file.csv"
parser = CSVParser()
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
# Display full document content
logger.error(document.content)
# Display individual chunks (rows)
for chunk in document.chunks:
logger.error(chunk.content)

View File

@ -89,11 +89,33 @@ class DocxParser(BaseParser):
logger.info(f"Setting max_workers to {max_workers} for document processing")
try:
inline_images: Dict[str, str] = {}
def _inline_upload(local_path: str) -> str:
"""Read temp image file, base64-encode, and return a ref path.
The Go-side ImageResolver (or main.py _resolve_images) handles
actual storage upload from Document.images.
"""
import base64
import uuid as _uuid
try:
with open(local_path, "rb") as f:
raw = f.read()
ext = os.path.splitext(local_path)[1].lower() or ".png"
ref = f"images/{_uuid.uuid4().hex}{ext}"
inline_images[ref] = base64.b64encode(raw).decode()
return ref
except Exception as exc:
logger.warning("Failed to read temp image %s: %s", local_path, exc)
return ""
logger.info(f"Starting Docx processing with max_pages={self.max_pages}")
docx_processor = Docx(
max_image_size=self.max_image_size,
enable_multimodal=self.enable_multimodal,
upload_file=self.storage.upload_file,
max_image_size=1920,
enable_multimodal=True,
upload_file=_inline_upload,
)
all_lines, tables = docx_processor(
binary=content,
@ -153,6 +175,7 @@ class DocxParser(BaseParser):
f"generated {len(text)} characters of text"
)
image_parts.update(inline_images)
return DocumentModel(content=text, images=image_parts)
except Exception as e:
logger.error(f"Error parsing DOCX document: {str(e)}")

View File

@ -5,44 +5,24 @@ import os
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
# Set up logger for this module
logger = logging.getLogger(__name__)
class ImageParser(BaseParser):
"""
Parser for image files with OCR capability.
Extracts text from images and generates captions.
"""Parser for standalone image files.
This parser handles image processing by:
1. Uploading the image to storage
2. Generating a descriptive caption
3. Performing OCR to extract text content
4. Returning a combined result with both text and image reference
Returns the image as a markdown reference with the raw image data
in Document.images so that the Go-side ImageResolver (or main.py's
_resolve_images) can handle storage upload.
"""
def parse_into_text(self, content: bytes) -> Document:
"""
Parse image content into markdown text
:param content: bytes content of the image
:return: Document object
"""
logger.info(f"Parsing image content, size: {len(content)} bytes")
logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
# Get file extension
ext = os.path.splitext(self.file_name)[1].lower()
ext = os.path.splitext(self.file_name)[1].lower() or ".png"
ref_path = f"images/{self.file_name}"
# Upload image to storage
image_url = self.storage.upload_bytes(content, file_ext=ext)
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
text = f"![{self.file_name}]({ref_path})"
images = {ref_path: base64.b64encode(content).decode()}
if not image_url:
logger.warning(f"Failed to upload image: {self.file_name}")
return Document(content=f"{self.file_name}", images={})
# Generate markdown text
text = f"![{self.file_name}]({image_url})"
images = {image_url: base64.b64encode(content).decode()}
# Create image object and add to map
return Document(content=text, images=images)

View File

@ -348,23 +348,11 @@ class MarkdownImageUtil:
class MarkdownImageBase64(BaseParser):
"""Parser for extracting and uploading base64 images from Markdown.
"""Parser for extracting base64 images from Markdown.
This parser:
1. Extracts base64-encoded images from Markdown content
2. Uploads the decoded images to storage
3. Replaces base64 data with uploaded URLs
4. Returns a Document with clean Markdown and image mappings
Requires:
- self.storage: Storage backend for uploading images
Example:
>>> parser = MarkdownImageBase64(storage=my_storage)
>>> content = b"![logo](data:image/png;base64,iVBORw0...)"
>>> doc = parser.parse_into_text(content)
>>> print(doc.content)
![logo](https://storage.com/uuid.png)
Extracts base64-encoded images, replaces them with path references,
and returns the raw image data in Document.images for the Go-side
ImageResolver (or main.py _resolve_images) to handle storage.
"""
def __init__(self, **kwargs):
@ -372,41 +360,14 @@ class MarkdownImageBase64(BaseParser):
self.image_helper = MarkdownImageUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse Markdown and process base64 images.
Args:
content: Raw Markdown content as bytes
Returns:
Document with:
- content: Markdown with base64 replaced by URLs
- images: Dict mapping URLs to base64 strings
"""
# Convert byte content to string using universal decoding method
text = endecode.decode_bytes(content)
# Extract base64 images and replace with temporary paths
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
# Final image mapping: URL -> base64 string (for Document)
images: Dict[str, str] = {}
# Temporary mapping: temp_path -> uploaded_URL (for replacement)
image_replace: Dict[str, str] = {}
for ipath, raw_bytes in img_b64.items():
images[ipath] = base64.b64encode(raw_bytes).decode()
logger.debug(f"Uploading {len(img_b64)} images from markdown")
# Upload each extracted image to storage
for ipath, b64_bytes in img_b64.items():
# Get file extension for proper MIME type
ext = os.path.splitext(ipath)[1].lower()
# Upload binary data and get back URL
image_url = self.storage.upload_bytes(b64_bytes, ext)
# Map temp path to uploaded URL for replacement
image_replace[ipath] = image_url
# Store base64 string in final images dict
images[image_url] = base64.b64encode(b64_bytes).decode()
# Replace temporary paths with actual uploaded URLs
text = self.image_helper.replace_path(text, image_replace)
logger.debug("Extracted %d base64 images from markdown", len(images))
return Document(content=text, images=images)

View File

@ -1,329 +0,0 @@
import logging
import re
import time
from typing import Dict, Optional
import markdownify
import requests
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_parser import MarkdownImageUtil, MarkdownTableFormatter
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class StdMinerUParser(BaseParser):
"""
Standard MinerU Parser for document parsing.
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
with support for tables, formulas, and images extraction.
"""
def __init__(
self,
enable_markdownify: bool = True,
mineru_endpoint: Optional[str] = None, # Added: 支持传入自定义 endpoint
**kwargs,
):
"""
Initialize MinerU parser.
Args:
enable_markdownify: Whether to convert HTML tables to markdown format
mineru_endpoint: MinerU API endpoint URL
**kwargs: Additional arguments passed to BaseParser
"""
super().__init__(**kwargs)
# Get MinerU endpoint from environment variable or parameter
# Modified: 优先使用传入的参数,否则使用 Config
base_url = mineru_endpoint if mineru_endpoint else CONFIG.mineru_endpoint
self.minerU = base_url.rstrip("/") if base_url else ""
self.enable_markdownify = enable_markdownify
# Helper for processing markdown images
self.image_helper = MarkdownImageUtil()
# Pattern to match base64 encoded images
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
# Check if MinerU API is available
self.enable = self.ping()
def ping(self, timeout: int = 5) -> bool:
"""
Check if MinerU API is available.
Args:
timeout: Request timeout in seconds
Returns:
True if API is available, False otherwise
"""
try:
response = requests.get(
self.minerU + "/docs", timeout=timeout, allow_redirects=True
)
response.raise_for_status()
return True
except Exception:
return False
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content into text using MinerU API.
Args:
content: Raw document content in bytes
Returns:
Document object containing parsed text and images
"""
if not self.enable:
logger.debug("MinerU API is not enabled")
return Document()
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
md_content: str = ""
images_b64: Dict[str, str] = {}
try:
# Call MinerU API to parse document
response = requests.post(
url=self.minerU + "/file_parse",
data={
"return_md": True, # Return markdown content
"return_images": True, # Return extracted images
"lang_list": ["ch", "en"], # Support Chinese and English
"table_enable": True, # Enable table parsing
"formula_enable": True, # Enable formula parsing
"parse_method": "auto", # Auto detect parsing method
"start_page_id": 0, # Start from first page
"end_page_id": 99999, # Parse all pages
"backend": "pipeline", # Use pipeline backend
"response_format_zip": False, # Return JSON instead of ZIP
"return_middle_json": False, # Don't return intermediate JSON
"return_model_output": False, # Don't return model output
"return_content_list": False, # Don't return content list
},
files={"files": content},
timeout=1000,
)
response.raise_for_status()
result = response.json()["results"]["files"]
md_content = result["md_content"]
images_b64 = result.get("images", {})
except Exception as e:
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
return Document()
# Convert HTML tables in markdown to markdown table format
if self.enable_markdownify:
logger.debug("Converting HTML to Markdown")
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
# Filter images that are actually used in markdown content
# Some images in images_b64 may not be referenced in md_content
# (e.g., images embedded in tables), so we need to filter them
for ipath, b64_str in images_b64.items():
# Skip images that are not referenced in markdown content
if f"images/{ipath}" not in md_content:
logger.debug(f"Image {ipath} not used in markdown")
continue
# Parse base64 image data
match = self.base64_pattern.match(b64_str)
if match:
# Extract image format (e.g., png, jpg)
file_ext = match.group(1)
# Extract base64 encoded data
b64_str = match.group(2)
# Decode base64 string to bytes
image_bytes = endecode.encode_image(b64_str, errors="ignore")
if not image_bytes:
logger.error("Failed to decode base64 image skip it")
continue
# Upload image to storage and get URL
image_url = self.storage.upload_bytes(
image_bytes, file_ext=f".{file_ext}"
)
# Store image mapping for later use
images[image_url] = b64_str
# Prepare replacement mapping for markdown content
image_replace[f"images/{ipath}"] = image_url
logger.info(f"Replaced {len(image_replace)} images in markdown")
# Replace image paths in markdown with uploaded URLs
text = self.image_helper.replace_path(md_content, image_replace)
logger.info(
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
)
return Document(content=text, images=images)
# Added: 新增 MinerUCloudParser 类,支持异步任务提交
class MinerUCloudParser(StdMinerUParser):
"""
MinerU Parser for REMOTE/CLOUD API (Asynchronous).
Uses the /submit -> /status -> /result workflow.
"""
SUBMIT_TIMEOUT = 30
POLL_INTERVAL = 2
MAX_WAIT_TIME = 600
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content using Cloud MinerU API (Async/Polling).
"""
if not self.enable:
return Document()
logger.info(f"Parsing PDF via Cloud MinerU API (size: {len(content)} bytes)")
try:
# --- Step 1: Submit Task ---
submit_url = f"{self.minerU}/submit"
logger.info(f"Submitting task to {submit_url}")
response = requests.post(
url=submit_url,
files={"files": content},
data={
"enable_formula": "true",
"enable_table": "true",
"layout_model": "doclayout_yolo",
"backend": "pipeline",
},
timeout=self.SUBMIT_TIMEOUT,
)
response.raise_for_status()
# Robust task_id extraction
resp_data = response.json()
task_id = resp_data.get("task_id") or resp_data.get("data", {}).get("task_id")
if not task_id:
raise ValueError(f"No task_id in response: {resp_data}")
logger.info(f"Task submitted, ID: {task_id}, waiting for completion...")
# --- Step 2: Poll Status ---
start_time = time.time()
while True:
if time.time() - start_time > self.MAX_WAIT_TIME:
raise TimeoutError(f"Task {task_id} timed out after {self.MAX_WAIT_TIME}s")
try:
status_resp = requests.get(
f"{self.minerU}/status/{task_id}",
timeout=10
)
status_resp.raise_for_status()
status_data = status_resp.json()
except requests.RequestException as e:
logger.warning(f"Status check failed for {task_id}: {e}. Retrying...")
time.sleep(self.POLL_INTERVAL)
continue
state = status_data.get("status") or status_data.get("state")
if state in ["done", "success"]:
break
elif state == "failed":
error_msg = status_data.get("error") or "Unknown error"
raise RuntimeError(f"Task {task_id} failed: {error_msg}")
else:
time.sleep(self.POLL_INTERVAL)
# --- Step 3: Get Result ---
result_resp = requests.get(
f"{self.minerU}/result/{task_id}",
timeout=30
)
result_resp.raise_for_status()
result_json = result_resp.json()
# Normalize result data
result_data = result_json.get("result", result_json)
md_content = result_data.get("md_content", "")
images_b64 = result_data.get("images", {})
# 使用父类的方法处理图片和Markdown转换 (复用现有逻辑)
# Convert HTML tables
if self.enable_markdownify:
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
for ipath, b64_str in images_b64.items():
if f"images/{ipath}" not in md_content:
continue
match = self.base64_pattern.match(b64_str)
if match:
file_ext = match.group(1)
b64_str_clean = match.group(2)
image_bytes = endecode.encode_image(b64_str_clean, errors="ignore")
if not image_bytes: continue
if self.storage:
image_url = self.storage.upload_bytes(image_bytes, file_ext=f".{file_ext}")
images[image_url] = b64_str_clean
image_replace[f"images/{ipath}"] = image_url
if image_replace:
md_content = self.image_helper.replace_path(md_content, image_replace)
return Document(content=md_content, images=images)
except Exception as e:
logger.error(f"Cloud MinerU parsing failed: {e}", exc_info=True)
return Document()
class MinerUParser(PipelineParser):
"""
MinerU Parser with pipeline processing.
This parser combines StdMinerUParser for document parsing and
MarkdownTableFormatter for table formatting in a pipeline.
"""
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
if __name__ == "__main__":
import os
# Example usage for testing
logging.basicConfig(level=logging.DEBUG)
# Configure your file path and MinerU endpoint
your_file = "/path/to/your/file.pdf"
# Added: 修改为 Localhost 方便测试
test_endpoint = "http://localhost:9987"
os.environ["MINERU_ENDPOINT"] = test_endpoint
# Create parser instance
# Modified: 传入 endpoint
parser = MinerUParser(mineru_endpoint=test_endpoint)
# Parse PDF file
if os.path.exists(your_file):
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
logger.error(document.content)
else:
print(f"File not found: {your_file}")

View File

@ -1,178 +1,82 @@
import logging
from typing import Dict, Type
from typing import Any, Optional
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.base_parser import BaseParser
from docreader.parser.csv_parser import CSVParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.pdf_parser import PDFParser
from docreader.parser.text_parser import TextParser
from docreader.parser.registry import registry
from docreader.parser.web_parser import WebParser
logger = logging.getLogger(__name__)
class Parser:
"""
Document parser facade that integrates all specialized parsers.
Provides a unified interface for parsing various document types.
"""Document parser facade (lightweight version).
Converts files/URLs to markdown + image references.
No chunking, no storage, no OCR, no VLM.
"""
def __init__(self):
# Initialize all parser types - maps file extensions to their corresponding parser classes
self.parsers: Dict[str, Type[BaseParser]] = {
# Document formats
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"txt": TextParser,
# Image formats - all use the same ImageParser
"jpg": ImageParser,
"jpeg": ImageParser,
"png": ImageParser,
"gif": ImageParser,
"bmp": ImageParser,
"tiff": ImageParser,
"webp": ImageParser,
# Alternative markdown extension
"markdown": MarkdownParser,
# Spreadsheet formats
"csv": CSVParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
}
self.registry = registry
logger.info(
"Parser initialized with %d parsers: %s",
len(self.parsers),
", ".join(self.parsers.keys()),
"Parser initialized with engines: %s",
", ".join(self.registry.get_engine_names()),
)
def get_parser(self, file_type: str) -> Type[BaseParser]:
"""
Get parser class for the specified file type.
Args:
file_type: The file extension or type identifier
Returns:
Parser class for the file type, or None if unsupported
"""
# Look up parser by file type (case-insensitive)
parser = self.parsers.get(file_type.lower())
if not parser:
# Raise error if file type is not supported
raise ValueError(f"Unsupported file type: {file_type}")
return parser
def parse_file(
self,
file_name: str,
file_type: str,
content: bytes,
config: ChunkingConfig,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
) -> Document:
"""
Parse file content using appropriate parser based on file type.
Args:
file_name: Name of the file being parsed
file_type: Type/extension of the file
content: Raw file content as bytes
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing file: {file_name} with type: {file_type}")
"""Parse file content to markdown."""
engine = parser_engine or ""
overrides = engine_overrides or {}
logger.info(
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
"Parsing file: %s, type: %s, engine: %s",
file_name,
file_type,
engine or "builtin",
)
# Get appropriate parser class for the file type
cls = self.get_parser(file_type)
# Create parser instance with configuration
logger.info(f"Creating parser instance for {file_type} file")
cls = self.registry.get_parser_class(engine, file_type)
logger.info(
"Creating %s parser instance for %s file",
cls.__name__,
file_type,
)
parser = cls(
file_name=file_name,
file_type=file_type,
chunk_size=config.chunk_size, # Size of each text chunk
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
**overrides,
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
# Execute the parsing process
logger.info("Starting to parse file content, size: %d bytes", len(content))
result = parser.parse(content)
# Validate parsing results and log warnings if needed
if not result.content:
logger.warning(f"Parser returned empty content for file: {file_name}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for file: {file_name}")
elif result.chunks[0]:
# Log first chunk size for debugging
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
logger.warning("Parser returned empty content for file: %s", file_name)
logger.info(
"Parsed file %s, content length=%d", file_name, len(result.content)
)
return result
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
"""
Parse content from a URL using the WebParser.
Args:
url: URL to parse
title: Title of the webpage (for metadata)
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing URL: {url}, title: {title}")
logger.info(
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
)
# Create web parser instance with configuration
logger.info("Creating WebParser instance")
parser = WebParser(
title=title, # Webpage title for metadata
chunk_size=config.chunk_size, # Size of each text chunk
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
def parse_url(
self,
url: str,
title: str,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
) -> Document:
"""Parse content from a URL to markdown."""
logger.info("Parsing URL: %s, title: %s", url, title)
parser = WebParser(title=title)
logger.info("Starting to parse URL content")
# Parse URL content (encode URL string to bytes as required by parser interface)
result = parser.parse(url.encode())
# Validate parsing results and log warnings if needed
if not result.content:
logger.warning(f"Parser returned empty content for url: {url}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for url: {url}")
elif result.chunks[0]:
# Log first chunk size for debugging
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
logger.warning("Parser returned empty content for url: %s", url)
logger.info("Parsed url %s, content length=%d", url, len(result.content))
return result

View File

@ -1,6 +1,5 @@
from docreader.parser.chain_parser import FirstParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.mineru_parser import MinerUParser
class PDFParser(FirstParser):
@ -13,4 +12,4 @@ class PDFParser(FirstParser):
The first successful parser result will be returned.
"""
# Parser classes to try in order (chain of responsibility pattern)
_parser_cls = (MinerUParser, MarkitdownParser)
_parser_cls = (MarkitdownParser,)

View File

@ -0,0 +1,160 @@
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple, Type
from docreader.parser.base_parser import BaseParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.pdf_parser import PDFParser
logger = logging.getLogger(__name__)
BUILTIN_ENGINE = "builtin"
class ParserEngineRegistry:
"""Registry for parser engines.
Each engine maps file extensions to parser classes.
When a requested engine doesn't support a file type, the registry
falls back to the builtin engine automatically.
"""
def __init__(self):
self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
self._descriptions: Dict[str, str] = {}
self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
self._unavailable_hint: Dict[str, str] = {}
def register(
self,
name: str,
file_types: Dict[str, Type[BaseParser]],
description: str = "",
check_available: Callable[..., Tuple[bool, str]] | None = None,
unavailable_hint: str = "",
):
self._engines[name] = file_types
self._descriptions[name] = description
if check_available is not None:
self._check_available[name] = check_available
self._unavailable_hint[name] = unavailable_hint
logger.info(
"Registered parser engine '%s' with file types: %s",
name,
", ".join(file_types.keys()),
)
def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
"""Resolve parser class for the given engine and file type.
Falls back to builtin engine when the requested engine doesn't
support the file type.
"""
ft = file_type.lower()
if engine and engine in self._engines:
cls = self._engines[engine].get(ft)
if cls:
logger.info("Using engine '%s' for file type '%s'", engine, ft)
return cls
logger.info(
"Engine '%s' does not support '%s', falling back to builtin",
engine,
ft,
)
builtin = self._engines.get(BUILTIN_ENGINE, {})
cls = builtin.get(ft)
if cls:
return cls
raise ValueError(f"Unsupported file type: {file_type}")
def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
"""Return metadata for all registered engines, including availability.
Args:
overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
forwarded to each engine's check_available function.
"""
result = []
for name, parsers in self._engines.items():
available = True
unavailable_reason = ""
check = self._check_available.get(name)
if check is not None:
try:
available, unavailable_reason = check(overrides)
except Exception as e:
available = False
unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
if not available and not unavailable_reason:
unavailable_reason = self._unavailable_hint.get(name, "不可用")
result.append(
{
"name": name,
"description": self._descriptions.get(name, ""),
"file_types": sorted(parsers.keys()),
"available": available,
"unavailable_reason": unavailable_reason,
}
)
return result
def get_engine_names(self) -> List[str]:
return list(self._engines.keys())
def _build_default_registry() -> ParserEngineRegistry:
"""Create and populate the default registry with all known engines."""
reg = ParserEngineRegistry()
_image_types = {
ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
}
reg.register(
BUILTIN_ENGINE,
{
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"markdown": MarkdownParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
**_image_types,
},
description="内置解析引擎",
)
reg.register(
"markitdown",
{
"md": MarkitdownParser,
"markdown": MarkitdownParser,
"pdf": MarkitdownParser,
"docx": MarkitdownParser,
"doc": MarkitdownParser,
"pptx": MarkitdownParser,
"ppt": MarkitdownParser,
"xlsx": MarkitdownParser,
"xls": MarkitdownParser,
"csv": MarkitdownParser,
},
description="MarkItDown 解析引擎(微软 MarkItDown 库)",
)
# NOTE: Engine listing is managed by Go-side engine registry
# (docparser.ListAllEngines). The Python list_engines method is kept for
# backward compatibility with the gRPC ListEngines RPC but the Go app
# no longer calls it. MinerU engines are handled natively by Go.
return reg
registry = _build_default_registry()

View File

@ -5,96 +5,70 @@ import os
import traceback
import uuid
from abc import ABC, abstractmethod
from typing import Dict
from typing import Dict, Optional
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
from docreader.config import CONFIG
from docreader.utils import endecode
logger = logging.getLogger(__name__)
def _cfg(storage_config: Optional[Dict], key: str, *env_keys: str, default: str = "") -> str:
"""Read a value from storage_config dict, falling back to env vars."""
if storage_config:
v = storage_config.get(key, "")
if v:
return str(v)
for ek in env_keys:
v = os.environ.get(ek, "")
if v:
return v
return default
class Storage(ABC):
"""Abstract base class for object storage operations"""
@abstractmethod
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
class CosStorage(Storage):
"""Tencent Cloud COS storage implementation"""
def __init__(self, storage_config=None):
"""Initialize COS storage
Args:
storage_config: Storage configuration
"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.client, self.bucket_name, self.region, self.prefix = (
self._init_cos_client()
)
def _init_cos_client(self):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available,
# otherwise fall back to environment variables
if self.storage_config and self.storage_config.get("access_key_id") != "":
cos_config = self.storage_config
secret_id = cos_config.get("access_key_id")
secret_key = cos_config.get("secret_access_key")
region = cos_config.get("region")
bucket_name = cos_config.get("bucket_name")
appid = cos_config.get("app_id")
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = CONFIG.cos_secret_id
secret_key = CONFIG.cos_secret_key
region = CONFIG.cos_region
bucket_name = CONFIG.cos_bucket_name
appid = CONFIG.cos_app_id
prefix = CONFIG.cos_path_prefix
enable_old_domain = CONFIG.cos_enable_old_domain
sc = self.storage_config
secret_id = _cfg(sc, "access_key_id", "COS_SECRET_ID")
secret_key = _cfg(sc, "secret_access_key", "COS_SECRET_KEY")
region = _cfg(sc, "region", "COS_REGION")
bucket_name = _cfg(sc, "bucket_name", "COS_BUCKET_NAME")
appid = _cfg(sc, "app_id", "COS_APP_ID")
prefix = _cfg(sc, "path_prefix", "COS_PATH_PREFIX")
enable_old_domain = os.environ.get("COS_ENABLE_OLD_DOMAIN", "").lower() in ("1", "true", "yes")
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, "
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
"Incomplete COS configuration: "
"secret_id=%s, region=%s, bucket=%s, appid=%s",
bool(secret_id), region, bucket_name, appid,
)
return None, None, None, None
# Initialize COS configuration
logger.info(
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
)
logger.info("Initializing COS client: region=%s, bucket=%s", region, bucket_name)
config = CosConfig(
Appid=appid,
Region=region,
@ -102,93 +76,50 @@ class CosStorage(Storage):
SecretKey=secret_key,
EnableOldDomain=enable_old_domain,
)
# Create client
client = CosS3Client(config)
return client, bucket_name, region, prefix
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
logger.error("Failed to initialize COS client: %s", e)
return None, None, None, None
def _get_download_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to COS: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_ext = os.path.splitext(file_path)[1]
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
self.client.upload_file(
Bucket=self.bucket_name,
LocalFilePath=file_path,
Key=object_key,
)
# Get file URL
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded file to COS: {file_url}")
logger.info("COS upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
logger.error("COS upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to Tencent Cloud COS
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
if not self.client:
return ""
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated object key: {object_key}")
self.client.put_object(
Bucket=self.bucket_name, Body=content, Key=object_key
)
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
logger.info("COS upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to COS: {str(e)}")
logger.error("COS upload_bytes failed: %s", e)
traceback.print_exc()
return ""
@ -196,67 +127,34 @@ class CosStorage(Storage):
class MinioStorage(Storage):
"""MinIO storage implementation"""
def __init__(self, storage_config=None):
"""Initialize MinIO storage
Args:
storage_config: Storage configuration
"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
self._init_minio_client()
)
def _init_minio_client(self):
"""Initialize MinIO client from environment variables or injected config.
If storage_config contains valid configuration, prefer those values
to override environment variables.
"""
try:
# Get configuration from storage_config with environment variables as fallback
# Each field can independently fall back to environment variables
access_key = (
self.storage_config.get("access_key_id")
if self.storage_config and self.storage_config.get("access_key_id")
else CONFIG.minio_access_key_id
)
secret_key = (
self.storage_config.get("secret_access_key")
if self.storage_config and self.storage_config.get("secret_access_key")
else CONFIG.minio_secret_access_key
)
bucket_name = (
self.storage_config.get("bucket_name")
if self.storage_config and self.storage_config.get("bucket_name")
else CONFIG.minio_bucket_name
)
path_prefix_raw = (
self.storage_config.get("path_prefix")
if self.storage_config and self.storage_config.get("path_prefix")
else CONFIG.minio_path_prefix
)
sc = self.storage_config
access_key = _cfg(sc, "access_key_id", "MINIO_ACCESS_KEY_ID")
secret_key = _cfg(sc, "secret_access_key", "MINIO_SECRET_ACCESS_KEY")
bucket_name = _cfg(sc, "bucket_name", "MINIO_BUCKET_NAME")
path_prefix_raw = _cfg(sc, "path_prefix", "MINIO_PATH_PREFIX")
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
endpoint = CONFIG.minio_endpoint
use_ssl = CONFIG.minio_use_ssl
endpoint = _cfg(sc, "endpoint", "MINIO_ENDPOINT")
use_ssl = os.environ.get("MINIO_USE_SSL", "").lower() in ("1", "true", "yes")
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error(
"Incomplete MinIO configuration, missing environment variables"
)
logger.error("Incomplete MinIO configuration")
return None, None, None, None, None
# Initialize client
client = Minio(
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
)
# Ensure bucket exists
found = client.bucket_exists(bucket_name)
if not found:
client.make_bucket(bucket_name)
# Set public read policy for the bucket
policy = (
"{"
'"Version":"2012-10-17",'
@ -273,51 +171,26 @@ class MinioStorage(Storage):
return client, bucket_name, use_ssl, endpoint, path_prefix
except Exception as e:
logger.error(f"Failed to initialize MinIO client: {str(e)}")
logger.error("Failed to initialize MinIO client: %s", e)
return None, None, None, None, None
def _get_download_url(self, object_key: str):
"""Construct a public URL for MinIO object.
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
# 1. Use public endpoint if provided
endpoint = CONFIG.minio_public_endpoint
if endpoint:
return f"{endpoint}/{self.bucket_name}/{object_key}"
# 2. Use SSL if enabled
if self.use_ssl:
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
# 3. Use HTTP default
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
public_endpoint = os.environ.get("MINIO_PUBLIC_ENDPOINT", "")
if public_endpoint:
return f"{public_endpoint}/{self.bucket_name}/{object_key}"
scheme = "https" if self.use_ssl else "http"
return f"{scheme}://{self.endpoint}/{self.bucket_name}/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to MinIO
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to MinIO: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated MinIO object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to MinIO")
with open(file_path, "rb") as file_data:
file_size = os.path.getsize(file_path)
self.client.put_object(
@ -327,38 +200,22 @@ class MinioStorage(Storage):
length=file_size,
content_type="application/octet-stream",
)
# Get file URL
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
logger.info("MinIO upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload file to MinIO: {str(e)}")
logger.error("MinIO upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to MinIO
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
if not self.client:
return ""
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated MinIO object key: {object_key}")
self.client.put_object(
self.bucket_name or "",
object_key,
@ -367,52 +224,67 @@ class MinioStorage(Storage):
content_type="application/octet-stream",
)
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
logger.info("MinIO upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
logger.error("MinIO upload_bytes failed: %s", e)
traceback.print_exc()
return ""
class LocalStorage(Storage):
"""Local file system storage implementation"""
"""Local file system storage implementation.
def __init__(self, storage_config: Dict[str, str] = {}):
self.storage_config = storage_config
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
self.image_dir = os.path.join(base_dir, "images")
Saves files under base_dir and returns web-accessible URL paths
(e.g. /files/images/uuid.jpg) so that the Go app can serve them.
"""
def __init__(self, storage_config: Optional[Dict] = None):
sc = storage_config or {}
self.base_dir = (
sc.get("base_dir")
or os.environ.get("LOCAL_STORAGE_BASE_DIR", "/data/files")
)
path_prefix = (sc.get("path_prefix") or "").strip().strip("/")
if path_prefix:
self.image_dir = os.path.join(self.base_dir, path_prefix, "images")
else:
self.image_dir = os.path.join(self.base_dir, "images")
self.url_prefix = (
sc.get("url_prefix")
or os.environ.get("LOCAL_STORAGE_URL_PREFIX", "/files")
)
os.makedirs(self.image_dir, exist_ok=True)
def _to_url(self, fpath: str) -> str:
if self.url_prefix:
rel = os.path.relpath(fpath, self.base_dir)
return f"{self.url_prefix}/{rel}"
return fpath
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to local storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to local storage: {len(content)} bytes")
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fname, "wb") as f:
fpath = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fpath, "wb") as f:
f.write(content)
return fname
url = self._to_url(fpath)
logger.info("Local storage saved: %s -> %s", fpath, url)
return url
class Base64Storage(Storage):
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to base64 storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
file_ext = file_ext.lstrip(".")
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
class DummyStorage(Storage):
"""Dummy storage implementation.
It is used in tests or in environments where object storage is disabled.
All upload methods return empty string.
"""
"""Dummy storage — all uploads return empty string."""
def upload_file(self, file_path: str) -> str:
return ""
@ -421,26 +293,30 @@ class DummyStorage(Storage):
return ""
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
"""Create a storage instance based on configuration or environment variables
def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
"""Create a storage instance based on storage_config dict.
Args:
storage_config: Storage configuration dictionary
Returns:
Storage instance
The ``provider`` key in storage_config determines the backend:
minio, cos, local, base64.
Falls back to STORAGE_TYPE env var, then ``local``.
"""
storage_type = CONFIG.storage_type
storage_type = ""
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
provider = str(storage_config.get("provider", "")).lower().strip()
if provider and provider not in ("unspecified", "storage_provider_unspecified"):
storage_type = provider
if not storage_type:
storage_type = os.environ.get("STORAGE_TYPE", "local").lower().strip()
logger.info("Creating %s storage instance", storage_type)
if storage_type == "minio":
return MinioStorage(storage_config)
elif storage_type == "cos":
return CosStorage(storage_config)
elif storage_type == "local":
return LocalStorage(storage_config or {})
return LocalStorage(storage_config)
elif storage_type == "base64":
return Base64Storage()
return DummyStorage()

View File

@ -1,55 +0,0 @@
import logging
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class TextParser(BaseParser):
"""
Text document parser for processing plain text files.
This parser handles text extraction and chunking from plain text documents.
"""
def parse_into_text(self, content: bytes) -> Document:
"""
Parse text document content by decoding bytes to string.
This is a straightforward parser that simply converts the binary content
to text using appropriate character encoding.
Args:
content: Raw document content as bytes
Returns:
Parsed text content as string
"""
logger.info(f"Parsing text document, content size: {len(content)} bytes")
text = endecode.decode_bytes(content)
logger.info(
f"Successfully parsed text document, extracted {len(text)} characters"
)
return Document(content=text)
if __name__ == "__main__":
logger = logging.getLogger(__name__)
# Sample text for testing
text = """## 标题1
![alt text](image.png)
## 标题2
![alt text](image2.png)
## 标题3
![alt text](image3.png)"""
logger.info(f"Test text content: {text}")
# Define separators for text splitting
seperators = ["\n\n", "\n", "。"]
parser = TextParser(separators=seperators)
logger.info("Splitting text into units")
units = parser._split_into_units(text)
logger.info(f"Split text into {len(units)} units")
logger.info(f"Units: {units}")

File diff suppressed because it is too large Load Diff

View File

@ -4,86 +4,56 @@ package docreader;
option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
// 文档读取服务
service DocReader {
// 从文件读取文档
rpc ReadFromFile(ReadFromFileRequest) returns (ReadResponse) {}
// 从URL读取文档
rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {}
}
// 对象存储提供方
enum StorageProvider {
STORAGE_PROVIDER_UNSPECIFIED = 0;
COS = 1; // 腾讯云 COS
MINIO = 2; // MinIO/S3 兼容
}
// 通用对象存储配置,兼容 COS 与 MinIO
message StorageConfig {
StorageProvider provider = 1; // 存储提供方
string region = 2; // 区域(COS 使用)
string bucket_name = 3; // 桶名
string access_key_id = 4; // 访问密钥 ID(MinIO/S3 使用)
string secret_access_key = 5; // 访问密钥 Secret(MinIO/S3 使用)
string app_id = 6; // 应用 ID(COS 使用)
string path_prefix = 7; // 路径前缀
}
// VLM 配置
message VLMConfig {
string model_name = 1; // VLM Model Name
string base_url = 2; // VLM Base URL
string api_key = 3; // VLM API Key
string interface_type = 4; // VLM Interface Type: "ollama" or "openai"
rpc Read(ReadRequest) returns (ReadResponse) {}
rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
}
message ReadConfig {
int32 chunk_size = 1; // 分块大小
int32 chunk_overlap = 2; // 分块重叠
repeated string separators = 3; // 分隔符
bool enable_multimodal = 4; // 多模态处理
StorageConfig storage_config = 5; // 对象存储配置(通用)
VLMConfig vlm_config = 6; // VLM 配置
string parser_engine = 1;
map<string, string> parser_engine_overrides = 2;
map<string, string> image_storage = 3; // storage provider config (provider, endpoint, bucket, ...)
}
// 从文件读取文档请求
message ReadFromFileRequest {
bytes file_content = 1; // 文件内容
string file_name = 2; // 文件名
string file_type = 3; // 文件类型
ReadConfig read_config = 4;
string request_id = 5;
// Unified read request: set file_content for file mode, url for URL mode.
message ReadRequest {
bytes file_content = 1;
string file_name = 2;
string file_type = 3;
string url = 4;
string title = 5;
ReadConfig config = 6;
string request_id = 7;
}
// 从URL读取文档请求
message ReadFromURLRequest {
string url = 1; // 文档URL
string title = 2; // 标题
ReadConfig read_config = 3;
string request_id = 4;
message ImageRef {
string filename = 1;
string original_ref = 2;
string mime_type = 3;
string storage_key = 4; // download URL from shared storage
bytes image_data = 5; // inline bytes fallback
}
// 图片信息
message Image {
string url = 1; // 图片URL
string caption = 2; // 图片描述
string ocr_text = 3; // OCR提取的文本内容
string original_url = 4; // 原始图片URL
int32 start = 5; // 图片在文本中的开始位置
int32 end = 6; // 图片在文本中的结束位置
}
message Chunk {
string content = 1; // 块内容
int32 seq = 2; // 块在文档中的次序
int32 start = 3; // 块在文档中的起始位置
int32 end = 4; // 块在文档中的结束位置
repeated Image images = 5; // 块中包含的图片信息
}
// 从URL读取文档响应
message ReadResponse {
repeated Chunk chunks = 1; // 文档分块
string error = 2; // 错误信息
}
string markdown_content = 1;
repeated ImageRef image_refs = 2;
string image_dir_path = 3;
map<string, string> metadata = 4;
string error = 5;
}
message ListEnginesRequest {
map<string, string> config_overrides = 1;
}
message ParserEngineInfo {
string name = 1;
string description = 2;
repeated string file_types = 3;
bool available = 4;
string unavailable_reason = 5;
}
message ListEnginesResponse {
repeated ParserEngineInfo engines = 1;
}

View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.5.1
// - protoc v5.29.3
// - protoc v6.33.4
// source: docreader.proto
package proto
@ -19,20 +19,18 @@ import (
const _ = grpc.SupportPackageIsVersion9
const (
DocReader_ReadFromFile_FullMethodName = "/docreader.DocReader/ReadFromFile"
DocReader_ReadFromURL_FullMethodName = "/docreader.DocReader/ReadFromURL"
DocReader_Read_FullMethodName = "/docreader.DocReader/Read"
DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
DocReader_ConvertToPDF_FullMethodName = "/docreader.DocReader/ConvertToPDF"
)
// DocReaderClient is the client API for DocReader service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// 文档读取服务
type DocReaderClient interface {
// 从文件读取文档
ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error)
// 从URL读取文档
ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error)
Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error)
ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error)
}
type docReaderClient struct {
@ -43,20 +41,30 @@ func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient {
return &docReaderClient{cc}
}
func (c *docReaderClient) ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
func (c *docReaderClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ReadResponse)
err := c.cc.Invoke(ctx, DocReader_ReadFromFile_FullMethodName, in, out, cOpts...)
err := c.cc.Invoke(ctx, DocReader_Read_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ReadResponse)
err := c.cc.Invoke(ctx, DocReader_ReadFromURL_FullMethodName, in, out, cOpts...)
out := new(ListEnginesResponse)
err := c.cc.Invoke(ctx, DocReader_ListEngines_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *docReaderClient) ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ConvertToPDFResponse)
err := c.cc.Invoke(ctx, DocReader_ConvertToPDF_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
@ -66,13 +74,10 @@ func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLReques
// DocReaderServer is the server API for DocReader service.
// All implementations must embed UnimplementedDocReaderServer
// for forward compatibility.
//
// 文档读取服务
type DocReaderServer interface {
// 从文件读取文档
ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error)
// 从URL读取文档
ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error)
Read(context.Context, *ReadRequest) (*ReadResponse, error)
ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error)
ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error)
mustEmbedUnimplementedDocReaderServer()
}
@ -83,11 +88,14 @@ type DocReaderServer interface {
// pointer dereference when methods are called.
type UnimplementedDocReaderServer struct{}
func (UnimplementedDocReaderServer) ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReadFromFile not implemented")
func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
}
func (UnimplementedDocReaderServer) ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReadFromURL not implemented")
func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented")
}
func (UnimplementedDocReaderServer) ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ConvertToPDF not implemented")
}
func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
func (UnimplementedDocReaderServer) testEmbeddedByValue() {}
@ -110,38 +118,56 @@ func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) {
s.RegisterService(&DocReader_ServiceDesc, srv)
}
func _DocReader_ReadFromFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadFromFileRequest)
func _DocReader_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ReadFromFile(ctx, in)
return srv.(DocReaderServer).Read(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ReadFromFile_FullMethodName,
FullMethod: DocReader_Read_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ReadFromFile(ctx, req.(*ReadFromFileRequest))
return srv.(DocReaderServer).Read(ctx, req.(*ReadRequest))
}
return interceptor(ctx, in, info, handler)
}
func _DocReader_ReadFromURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadFromURLRequest)
func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListEnginesRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ReadFromURL(ctx, in)
return srv.(DocReaderServer).ListEngines(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ReadFromURL_FullMethodName,
FullMethod: DocReader_ListEngines_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ReadFromURL(ctx, req.(*ReadFromURLRequest))
return srv.(DocReaderServer).ListEngines(ctx, req.(*ListEnginesRequest))
}
return interceptor(ctx, in, info, handler)
}
func _DocReader_ConvertToPDF_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ConvertToPDFRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ConvertToPDF(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ConvertToPDF_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ConvertToPDF(ctx, req.(*ConvertToPDFRequest))
}
return interceptor(ctx, in, info, handler)
}
@ -154,12 +180,16 @@ var DocReader_ServiceDesc = grpc.ServiceDesc{
HandlerType: (*DocReaderServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "ReadFromFile",
Handler: _DocReader_ReadFromFile_Handler,
MethodName: "Read",
Handler: _DocReader_Read_Handler,
},
{
MethodName: "ReadFromURL",
Handler: _DocReader_ReadFromURL_Handler,
MethodName: "ListEngines",
Handler: _DocReader_ListEngines_Handler,
},
{
MethodName: "ConvertToPDF",
Handler: _DocReader_ConvertToPDF_Handler,
},
},
Streams: []grpc.StreamDesc{},

View File

@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xb9\x01\n\rStorageConfig\x12,\n\x08provider\x18\x01 \x01(\x0e\x32\x1a.docreader.StorageProvider\x12\x0e\n\x06region\x18\x02 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x03 \x01(\t\x12\x15\n\raccess_key_id\x18\x04 \x01(\t\x12\x19\n\x11secret_access_key\x18\x05 \x01(\t\x12\x0e\n\x06\x61pp_id\x18\x06 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x07 \x01(\t\"Z\n\tVLMConfig\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x10\n\x08\x62\x61se_url\x18\x02 \x01(\t\x12\x0f\n\x07\x61pi_key\x18\x03 \x01(\t\x12\x16\n\x0einterface_type\x18\x04 \x01(\t\"\xc2\x01\n\nReadConfig\x12\x12\n\nchunk_size\x18\x01 \x01(\x05\x12\x15\n\rchunk_overlap\x18\x02 \x01(\x05\x12\x12\n\nseparators\x18\x03 \x03(\t\x12\x19\n\x11\x65nable_multimodal\x18\x04 \x01(\x08\x12\x30\n\x0estorage_config\x18\x05 \x01(\x0b\x32\x18.docreader.StorageConfig\x12(\n\nvlm_config\x18\x06 \x01(\x0b\x32\x14.docreader.VLMConfig\"\x91\x01\n\x13ReadFromFileRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12*\n\x0bread_config\x18\x04 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"p\n\x12ReadFromURLRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12*\n\x0bread_config\x18\x03 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"i\n\x05Image\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x61ption\x18\x02 \x01(\t\x12\x10\n\x08ocr_text\x18\x03 \x01(\t\x12\x14\n\x0coriginal_url\x18\x04 \x01(\t\x12\r\n\x05start\x18\x05 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x06 \x01(\x05\"c\n\x05\x43hunk\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x05\x12\r\n\x05start\x18\x03 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x05\x12 \n\x06images\x18\x05 \x03(\x0b\x32\x10.docreader.Image\"?\n\x0cReadResponse\x12 \n\x06\x63hunks\x18\x01 \x03(\x0b\x32\x10.docreader.Chunk\x12\r\n\x05\x65rror\x18\x02 \x01(\t*G\n\x0fStorageProvider\x12 \n\x1cSTORAGE_PROVIDER_UNSPECIFIED\x10\x00\x12\x07\n\x03\x43OS\x10\x01\x12\t\n\x05MINIO\x10\x02\x32\x9f\x01\n\tDocReader\x12I\n\x0cReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\x0bReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xa9\x02\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x12>\n\rimage_storage\x18\x03 \x03(\x0b\x32\'.docreader.ReadConfig.ImageStorageEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11ImageStorageEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo\"Q\n\x13\x43onvertToPDFRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\":\n\x14\x43onvertToPDFResponse\x12\x13\n\x0bpdf_content\x18\x01 \x01(\x0c\x12\r\n\x05\x65rror\x18\x02 \x01(\t2\xe9\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x12Q\n\x0c\x43onvertToPDF\x12\x1e.docreader.ConvertToPDFRequest\x1a\x1f.docreader.ConvertToPDFResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@ -32,24 +32,40 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docreader_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
_globals['_STORAGEPROVIDER']._serialized_start=1042
_globals['_STORAGEPROVIDER']._serialized_end=1113
_globals['_STORAGECONFIG']._serialized_start=31
_globals['_STORAGECONFIG']._serialized_end=216
_globals['_VLMCONFIG']._serialized_start=218
_globals['_VLMCONFIG']._serialized_end=308
_globals['_READCONFIG']._serialized_start=311
_globals['_READCONFIG']._serialized_end=505
_globals['_READFROMFILEREQUEST']._serialized_start=508
_globals['_READFROMFILEREQUEST']._serialized_end=653
_globals['_READFROMURLREQUEST']._serialized_start=655
_globals['_READFROMURLREQUEST']._serialized_end=767
_globals['_IMAGE']._serialized_start=769
_globals['_IMAGE']._serialized_end=874
_globals['_CHUNK']._serialized_start=876
_globals['_CHUNK']._serialized_end=975
_globals['_READRESPONSE']._serialized_start=977
_globals['_READRESPONSE']._serialized_end=1040
_globals['_DOCREADER']._serialized_start=1116
_globals['_DOCREADER']._serialized_end=1275
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
_globals['_READCONFIG_IMAGESTORAGEENTRY']._loaded_options = None
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_options = b'8\001'
_globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
_globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
_globals['_READCONFIG']._serialized_start=31
_globals['_READCONFIG']._serialized_end=328
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=215
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=275
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_start=277
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_end=328
_globals['_READREQUEST']._serialized_start=331
_globals['_READREQUEST']._serialized_end=491
_globals['_IMAGEREF']._serialized_start=493
_globals['_IMAGEREF']._serialized_end=603
_globals['_READRESPONSE']._serialized_start=606
_globals['_READRESPONSE']._serialized_end=832
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=785
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=832
_globals['_LISTENGINESREQUEST']._serialized_start=835
_globals['_LISTENGINESREQUEST']._serialized_end=989
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=935
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=989
_globals['_PARSERENGINEINFO']._serialized_start=991
_globals['_PARSERENGINEINFO']._serialized_end=1111
_globals['_LISTENGINESRESPONSE']._serialized_start=1113
_globals['_LISTENGINESRESPONSE']._serialized_end=1180
_globals['_CONVERTTOPDFREQUEST']._serialized_start=1182
_globals['_CONVERTTOPDFREQUEST']._serialized_end=1263
_globals['_CONVERTTOPDFRESPONSE']._serialized_start=1265
_globals['_CONVERTTOPDFRESPONSE']._serialized_end=1323
_globals['_DOCREADER']._serialized_start=1326
_globals['_DOCREADER']._serialized_end=1559
# @@protoc_insertion_point(module_scope)

View File

@ -1,5 +1,4 @@
from google.protobuf.internal import containers as _containers
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from collections.abc import Iterable as _Iterable, Mapping as _Mapping
@ -7,121 +6,130 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
DESCRIPTOR: _descriptor.FileDescriptor
class StorageProvider(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
__slots__ = ()
STORAGE_PROVIDER_UNSPECIFIED: _ClassVar[StorageProvider]
COS: _ClassVar[StorageProvider]
MINIO: _ClassVar[StorageProvider]
STORAGE_PROVIDER_UNSPECIFIED: StorageProvider
COS: StorageProvider
MINIO: StorageProvider
class StorageConfig(_message.Message):
__slots__ = ("provider", "region", "bucket_name", "access_key_id", "secret_access_key", "app_id", "path_prefix")
PROVIDER_FIELD_NUMBER: _ClassVar[int]
REGION_FIELD_NUMBER: _ClassVar[int]
BUCKET_NAME_FIELD_NUMBER: _ClassVar[int]
ACCESS_KEY_ID_FIELD_NUMBER: _ClassVar[int]
SECRET_ACCESS_KEY_FIELD_NUMBER: _ClassVar[int]
APP_ID_FIELD_NUMBER: _ClassVar[int]
PATH_PREFIX_FIELD_NUMBER: _ClassVar[int]
provider: StorageProvider
region: str
bucket_name: str
access_key_id: str
secret_access_key: str
app_id: str
path_prefix: str
def __init__(self, provider: _Optional[_Union[StorageProvider, str]] = ..., region: _Optional[str] = ..., bucket_name: _Optional[str] = ..., access_key_id: _Optional[str] = ..., secret_access_key: _Optional[str] = ..., app_id: _Optional[str] = ..., path_prefix: _Optional[str] = ...) -> None: ...
class VLMConfig(_message.Message):
__slots__ = ("model_name", "base_url", "api_key", "interface_type")
MODEL_NAME_FIELD_NUMBER: _ClassVar[int]
BASE_URL_FIELD_NUMBER: _ClassVar[int]
API_KEY_FIELD_NUMBER: _ClassVar[int]
INTERFACE_TYPE_FIELD_NUMBER: _ClassVar[int]
model_name: str
base_url: str
api_key: str
interface_type: str
def __init__(self, model_name: _Optional[str] = ..., base_url: _Optional[str] = ..., api_key: _Optional[str] = ..., interface_type: _Optional[str] = ...) -> None: ...
class ReadConfig(_message.Message):
__slots__ = ("chunk_size", "chunk_overlap", "separators", "enable_multimodal", "storage_config", "vlm_config")
CHUNK_SIZE_FIELD_NUMBER: _ClassVar[int]
CHUNK_OVERLAP_FIELD_NUMBER: _ClassVar[int]
SEPARATORS_FIELD_NUMBER: _ClassVar[int]
ENABLE_MULTIMODAL_FIELD_NUMBER: _ClassVar[int]
STORAGE_CONFIG_FIELD_NUMBER: _ClassVar[int]
VLM_CONFIG_FIELD_NUMBER: _ClassVar[int]
chunk_size: int
chunk_overlap: int
separators: _containers.RepeatedScalarFieldContainer[str]
enable_multimodal: bool
storage_config: StorageConfig
vlm_config: VLMConfig
def __init__(self, chunk_size: _Optional[int] = ..., chunk_overlap: _Optional[int] = ..., separators: _Optional[_Iterable[str]] = ..., enable_multimodal: bool = ..., storage_config: _Optional[_Union[StorageConfig, _Mapping]] = ..., vlm_config: _Optional[_Union[VLMConfig, _Mapping]] = ...) -> None: ...
__slots__ = ("parser_engine", "parser_engine_overrides", "image_storage")
class ParserEngineOverridesEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
class ImageStorageEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
PARSER_ENGINE_FIELD_NUMBER: _ClassVar[int]
PARSER_ENGINE_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
IMAGE_STORAGE_FIELD_NUMBER: _ClassVar[int]
parser_engine: str
parser_engine_overrides: _containers.ScalarMap[str, str]
image_storage: _containers.ScalarMap[str, str]
def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ..., image_storage: _Optional[_Mapping[str, str]] = ...) -> None: ...
class ReadFromFileRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type", "read_config", "request_id")
class ReadRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type", "url", "title", "config", "request_id")
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
URL_FIELD_NUMBER: _ClassVar[int]
TITLE_FIELD_NUMBER: _ClassVar[int]
CONFIG_FIELD_NUMBER: _ClassVar[int]
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
file_content: bytes
file_name: str
file_type: str
read_config: ReadConfig
request_id: str
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
class ReadFromURLRequest(_message.Message):
__slots__ = ("url", "title", "read_config", "request_id")
URL_FIELD_NUMBER: _ClassVar[int]
TITLE_FIELD_NUMBER: _ClassVar[int]
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
url: str
title: str
read_config: ReadConfig
config: ReadConfig
request_id: str
def __init__(self, url: _Optional[str] = ..., title: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., url: _Optional[str] = ..., title: _Optional[str] = ..., config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
class Image(_message.Message):
__slots__ = ("url", "caption", "ocr_text", "original_url", "start", "end")
URL_FIELD_NUMBER: _ClassVar[int]
CAPTION_FIELD_NUMBER: _ClassVar[int]
OCR_TEXT_FIELD_NUMBER: _ClassVar[int]
ORIGINAL_URL_FIELD_NUMBER: _ClassVar[int]
START_FIELD_NUMBER: _ClassVar[int]
END_FIELD_NUMBER: _ClassVar[int]
url: str
caption: str
ocr_text: str
original_url: str
start: int
end: int
def __init__(self, url: _Optional[str] = ..., caption: _Optional[str] = ..., ocr_text: _Optional[str] = ..., original_url: _Optional[str] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ...
class Chunk(_message.Message):
__slots__ = ("content", "seq", "start", "end", "images")
CONTENT_FIELD_NUMBER: _ClassVar[int]
SEQ_FIELD_NUMBER: _ClassVar[int]
START_FIELD_NUMBER: _ClassVar[int]
END_FIELD_NUMBER: _ClassVar[int]
IMAGES_FIELD_NUMBER: _ClassVar[int]
content: str
seq: int
start: int
end: int
images: _containers.RepeatedCompositeFieldContainer[Image]
def __init__(self, content: _Optional[str] = ..., seq: _Optional[int] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., images: _Optional[_Iterable[_Union[Image, _Mapping]]] = ...) -> None: ...
class ImageRef(_message.Message):
__slots__ = ("filename", "original_ref", "mime_type", "storage_key", "image_data")
FILENAME_FIELD_NUMBER: _ClassVar[int]
ORIGINAL_REF_FIELD_NUMBER: _ClassVar[int]
MIME_TYPE_FIELD_NUMBER: _ClassVar[int]
STORAGE_KEY_FIELD_NUMBER: _ClassVar[int]
IMAGE_DATA_FIELD_NUMBER: _ClassVar[int]
filename: str
original_ref: str
mime_type: str
storage_key: str
image_data: bytes
def __init__(self, filename: _Optional[str] = ..., original_ref: _Optional[str] = ..., mime_type: _Optional[str] = ..., storage_key: _Optional[str] = ..., image_data: _Optional[bytes] = ...) -> None: ...
class ReadResponse(_message.Message):
__slots__ = ("chunks", "error")
CHUNKS_FIELD_NUMBER: _ClassVar[int]
__slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error")
class MetadataEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int]
IMAGE_REFS_FIELD_NUMBER: _ClassVar[int]
IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
METADATA_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
chunks: _containers.RepeatedCompositeFieldContainer[Chunk]
markdown_content: str
image_refs: _containers.RepeatedCompositeFieldContainer[ImageRef]
image_dir_path: str
metadata: _containers.ScalarMap[str, str]
error: str
def __init__(self, chunks: _Optional[_Iterable[_Union[Chunk, _Mapping]]] = ..., error: _Optional[str] = ...) -> None: ...
def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ...
class ListEnginesRequest(_message.Message):
__slots__ = ("config_overrides",)
class ConfigOverridesEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
CONFIG_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
config_overrides: _containers.ScalarMap[str, str]
def __init__(self, config_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
class ParserEngineInfo(_message.Message):
__slots__ = ("name", "description", "file_types", "available", "unavailable_reason")
NAME_FIELD_NUMBER: _ClassVar[int]
DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
FILE_TYPES_FIELD_NUMBER: _ClassVar[int]
AVAILABLE_FIELD_NUMBER: _ClassVar[int]
UNAVAILABLE_REASON_FIELD_NUMBER: _ClassVar[int]
name: str
description: str
file_types: _containers.RepeatedScalarFieldContainer[str]
available: bool
unavailable_reason: str
def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., file_types: _Optional[_Iterable[str]] = ..., available: bool = ..., unavailable_reason: _Optional[str] = ...) -> None: ...
class ListEnginesResponse(_message.Message):
__slots__ = ("engines",)
ENGINES_FIELD_NUMBER: _ClassVar[int]
engines: _containers.RepeatedCompositeFieldContainer[ParserEngineInfo]
def __init__(self, engines: _Optional[_Iterable[_Union[ParserEngineInfo, _Mapping]]] = ...) -> None: ...
class ConvertToPDFRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type")
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
file_content: bytes
file_name: str
file_type: str
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ...) -> None: ...
class ConvertToPDFResponse(_message.Message):
__slots__ = ("pdf_content", "error")
PDF_CONTENT_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
pdf_content: bytes
error: str
def __init__(self, pdf_content: _Optional[bytes] = ..., error: _Optional[str] = ...) -> None: ...

View File

@ -3,9 +3,9 @@
import grpc
import warnings
import docreader_pb2 as docreader__pb2
from docreader.proto import docreader_pb2 as docreader__pb2
GRPC_GENERATED_VERSION = '1.76.0'
GRPC_GENERATED_VERSION = '1.78.0'
GRPC_VERSION = grpc.__version__
_version_not_supported = False
@ -26,8 +26,7 @@ if _version_not_supported:
class DocReaderStub(object):
"""文档读取服务
"""
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
@ -35,32 +34,40 @@ class DocReaderStub(object):
Args:
channel: A grpc.Channel.
"""
self.ReadFromFile = channel.unary_unary(
'/docreader.DocReader/ReadFromFile',
request_serializer=docreader__pb2.ReadFromFileRequest.SerializeToString,
self.Read = channel.unary_unary(
'/docreader.DocReader/Read',
request_serializer=docreader__pb2.ReadRequest.SerializeToString,
response_deserializer=docreader__pb2.ReadResponse.FromString,
_registered_method=True)
self.ReadFromURL = channel.unary_unary(
'/docreader.DocReader/ReadFromURL',
request_serializer=docreader__pb2.ReadFromURLRequest.SerializeToString,
response_deserializer=docreader__pb2.ReadResponse.FromString,
self.ListEngines = channel.unary_unary(
'/docreader.DocReader/ListEngines',
request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString,
response_deserializer=docreader__pb2.ListEnginesResponse.FromString,
_registered_method=True)
self.ConvertToPDF = channel.unary_unary(
'/docreader.DocReader/ConvertToPDF',
request_serializer=docreader__pb2.ConvertToPDFRequest.SerializeToString,
response_deserializer=docreader__pb2.ConvertToPDFResponse.FromString,
_registered_method=True)
class DocReaderServicer(object):
"""文档读取服务
"""
"""Missing associated documentation comment in .proto file."""
def ReadFromFile(self, request, context):
"""从文件读取文档
"""
def Read(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def ReadFromURL(self, request, context):
"""从URL读取文档
"""
def ListEngines(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def ConvertToPDF(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
@ -68,15 +75,20 @@ class DocReaderServicer(object):
def add_DocReaderServicer_to_server(servicer, server):
rpc_method_handlers = {
'ReadFromFile': grpc.unary_unary_rpc_method_handler(
servicer.ReadFromFile,
request_deserializer=docreader__pb2.ReadFromFileRequest.FromString,
'Read': grpc.unary_unary_rpc_method_handler(
servicer.Read,
request_deserializer=docreader__pb2.ReadRequest.FromString,
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
),
'ReadFromURL': grpc.unary_unary_rpc_method_handler(
servicer.ReadFromURL,
request_deserializer=docreader__pb2.ReadFromURLRequest.FromString,
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
'ListEngines': grpc.unary_unary_rpc_method_handler(
servicer.ListEngines,
request_deserializer=docreader__pb2.ListEnginesRequest.FromString,
response_serializer=docreader__pb2.ListEnginesResponse.SerializeToString,
),
'ConvertToPDF': grpc.unary_unary_rpc_method_handler(
servicer.ConvertToPDF,
request_deserializer=docreader__pb2.ConvertToPDFRequest.FromString,
response_serializer=docreader__pb2.ConvertToPDFResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
@ -87,11 +99,10 @@ def add_DocReaderServicer_to_server(servicer, server):
# This class is part of an EXPERIMENTAL API.
class DocReader(object):
"""文档读取服务
"""
"""Missing associated documentation comment in .proto file."""
@staticmethod
def ReadFromFile(request,
def Read(request,
target,
options=(),
channel_credentials=None,
@ -104,8 +115,8 @@ class DocReader(object):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ReadFromFile',
docreader__pb2.ReadFromFileRequest.SerializeToString,
'/docreader.DocReader/Read',
docreader__pb2.ReadRequest.SerializeToString,
docreader__pb2.ReadResponse.FromString,
options,
channel_credentials,
@ -118,7 +129,7 @@ class DocReader(object):
_registered_method=True)
@staticmethod
def ReadFromURL(request,
def ListEngines(request,
target,
options=(),
channel_credentials=None,
@ -131,9 +142,36 @@ class DocReader(object):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ReadFromURL',
docreader__pb2.ReadFromURLRequest.SerializeToString,
docreader__pb2.ReadResponse.FromString,
'/docreader.DocReader/ListEngines',
docreader__pb2.ListEnginesRequest.SerializeToString,
docreader__pb2.ListEnginesResponse.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)
@staticmethod
def ConvertToPDF(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ConvertToPDF',
docreader__pb2.ConvertToPDFRequest.SerializeToString,
docreader__pb2.ConvertToPDFResponse.FromString,
options,
channel_credentials,
insecure,

View File

@ -1,5 +1,5 @@
#!/bin/bash
set -x
set -ex
# 设置目录
PROTO_DIR="docreader/proto"
@ -13,12 +13,16 @@ python3 -m grpc_tools.protoc -I${PROTO_DIR} \
--grpc_python_out=${PYTHON_OUT} \
${PROTO_DIR}/docreader.proto
# 生成Go代码
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
--go_opt=paths=source_relative \
--go-grpc_out=${GO_OUT} \
--go-grpc_opt=paths=source_relative \
${PROTO_DIR}/docreader.proto
# 生成Go代码(仅在 protoc-gen-go 可用时执行)
if command -v protoc-gen-go &> /dev/null; then
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
--go_opt=paths=source_relative \
--go-grpc_out=${GO_OUT} \
--go-grpc_opt=paths=source_relative \
${PROTO_DIR}/docreader.proto
else
echo "protoc-gen-go not found, skipping Go code generation"
fi
# 修复Python导入问题(MacOS兼容版本)
if [ "$(uname)" == "Darwin" ]; then

165
docs/LITE.md 100644
View File

@ -0,0 +1,165 @@
# WeKnora Lite
零外部依赖的单二进制部署模式。无需 Docker、PostgreSQL、Redis,适合快速体验和小规模私有部署。
## 架构
| 组件 | 标准版 | Lite 版 |
|------|--------|---------|
| 数据库 | PostgreSQL | SQLite (WAL) |
| 向量检索 | pgvector / Qdrant / ES | sqlite-vec (vec0) |
| 关键词检索 | ParadeDB BM25 / ES | SQLite FTS5 |
| 消息队列 | Redis + Asynq | 内存 SyncTaskExecutor |
| 会话存储 | Redis | 内存 |
| 流管理 | Redis / 内存 | 内存 |
| 文件存储 | MinIO / COS / 本地 | 本地 |
| 文档解析 | DocReader (gRPC) | 不可用(文本/段落导入可用)|
| 前端 | Nginx 容器 | Go 内置静态文件服务 |
## 快速开始
### 方式一:Homebrew 安装(macOS / Linux,推荐)
```bash
brew tap Tencent/weknora https://github.com/Tencent/WeKnora
brew install weknora-lite
```
安装完成后,推荐使用 **brew services** 以后台服务方式运行:
```bash
brew services start weknora-lite # 启动服务(开机自动启动)
brew services info weknora-lite # 查看运行状态
# 首次启动自动创建配置文件 ~/.config/weknora/.env.lite
# 数据存储在 ~/.local/share/weknora/
# 访问 http://localhost:8080
```
常用服务管理命令:
```bash
brew services stop weknora-lite # 停止服务
brew services restart weknora-lite # 重启服务(修改配置后需重启)
brew services info weknora-lite # 查看状态
```
日志位于 `$(brew --prefix)/var/log/weknora-lite.log`。
也可以前台直接运行:
```bash
weknora-lite
```
如需修改配置(LLM 服务地址、安全密钥等):
```bash
$EDITOR ~/.config/weknora/.env.lite
brew services restart weknora-lite # 修改配置后重启生效
```
> **LLM 服务**:WeKnora Lite 需要一个 OpenAI 兼容的 LLM 服务来提供对话和 Embedding 能力。
> 可以使用 [Ollama](https://ollama.com/)(本地)、通义千问、OpenAI 等任何兼容服务,
> 在配置文件中设置对应的地址和 API Key 即可。
### 方式二:下载预编译包
从 [GitHub Releases](https://github.com/Tencent/WeKnora/releases) 下载对应平台的 tarball:
| 文件 | 平台 |
|------|------|
| `WeKnora-lite_*_linux_amd64.tar.gz` | Linux x86_64 |
| `WeKnora-lite_*_linux_arm64.tar.gz` | Linux ARM64 |
| `WeKnora-lite_*_darwin_amd64.tar.gz` | macOS Intel |
| `WeKnora-lite_*_darwin_arm64.tar.gz` | macOS Apple Silicon |
```bash
# 1. 解压
tar xzf WeKnora-lite_v0.2.0_darwin_arm64.tar.gz
cd WeKnora-lite_v0.2.0_darwin_arm64
# 2. 配置
cp .env.lite.example .env.lite
# 编辑 .env.lite,配置 LLM 服务地址和安全密钥
# 3. 运行
set -a && source .env.lite && set +a
./WeKnora-lite
# 访问 http://localhost:8080
```
### 方式三:从源码构建
前置条件:Go 1.22+(需要 CGO)、C 编译器 (gcc/clang)、Node.js 22+(前端构建)。
```bash
make run-lite
```
## 配置
Lite 模式通过 `.env.lite` 文件配置(模板见 `.env.lite.example`)。关键环境变量:
```bash
DB_DRIVER=sqlite # 使用 SQLite
DB_PATH=./data/weknora.db # 数据库文件路径
RETRIEVE_DRIVER=sqlite # SQLite 检索引擎 (FTS5 + sqlite-vec)
STORAGE_TYPE=local # 本地文件存储
LOCAL_STORAGE_BASE_DIR=./data/files
STREAM_MANAGER_TYPE=memory # 内存流管理
# REDIS_ADDR= # 留空 = 不使用 Redis
OLLAMA_BASE_URL=http://127.0.0.1:11434
```
完整配置参见 [.env.lite.example](../.env.lite.example)。
## 后台运行
### Homebrew 用户(macOS / Linux)
Homebrew 安装后直接使用 `brew services` 管理,详见上方「快速开始 → 方式一」。
### Linux systemd(tarball 安装)
tarball 中附带 `weknora-lite.service` 模板,按需修改路径后安装:
```bash
# 创建用户和目录
sudo useradd -r -s /sbin/nologin weknora
sudo mkdir -p /opt/weknora/data
sudo cp WeKnora-lite web/ .env.lite /opt/weknora/
sudo chown -R weknora:weknora /opt/weknora
# 安装并启动服务
sudo cp weknora-lite.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now weknora-lite
# 管理
sudo systemctl status weknora-lite # 查看状态
sudo journalctl -u weknora-lite -f # 查看日志
```
## 功能限制
与标准版相比,Lite 版有以下限制:
- **文档解析**:不支持文件上传和 URL 导入的自动解析(PDF/Word/Excel 等)。可使用文本和段落方式手动导入。
- **向量检索**:sqlite-vec 使用精确 KNN(非近似),适合 10 万条以下的小规模数据集。
- **并发**:SQLite 单写者模型,高并发写入场景下性能不如 PostgreSQL。
- **任务队列**:无持久化队列,进程重启后未完成的异步任务会丢失。
- **知识图谱**:默认禁用 (`NEO4J_ENABLE=false`)。
- **Agent Skills 沙箱**:默认禁用 (`WEKNORA_SANDBOX_MODE=disabled`)。
## 数据目录
默认所有数据存储在 `./data/` 目录下:
```
data/
├── weknora.db # SQLite 数据库
├── weknora.db-wal # WAL 日志
└── files/ # 上传文件
```
备份只需复制整个 `data/` 目录。

View File

@ -21,6 +21,15 @@ server {
try_files $uri $uri/ /index.html;
}
# 本地存储文件代理到后端服务(用于渲染 markdown 中的图片)
location /files/ {
proxy_pass ${APP_SCHEME}://${APP_HOST}:${APP_PORT}/files/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# API请求代理到后端服务
# APP_SCHEME 默认 http,远程 HTTPS 后端可设为 https
location /api/ {

File diff suppressed because it is too large Load Diff

View File

@ -15,7 +15,9 @@
"@microsoft/fetch-event-source": "^2.0.1",
"@types/dompurify": "^3.0.5",
"@types/papaparse": "^5.5.0",
"@vue-office/pptx": "^1.0.1",
"axios": "^1.8.4",
"docx-preview": "^0.3.7",
"dompurify": "^3.2.6",
"highlight.js": "^11.11.1",
"marked": "^5.1.2",
@ -27,6 +29,7 @@
"tdesign-icons-vue-next": "^0.4.1",
"tdesign-vue-next": "^1.17.2",
"vue": "^3.5.13",
"vue-demi": "^0.14.6",
"vue-i18n": "^11.1.12",
"vue-router": "^4.5.0",
"webpack": "^5.94.0",
@ -41,10 +44,10 @@
"@vue/tsconfig": "^0.7.0",
"less": "^4.3.0",
"less-loader": "^12.2.0",
"npm-run-all2": "^7.0.2",
"npm-run-all2": "^8.0.4",
"typescript": "~5.8.0",
"vite": "^7.2.2",
"vue-tsc": "^2.2.8"
"vue-tsc": "^3.2.5"
},
"overrides": {
"lightningcss": "none",

View File

@ -82,24 +82,13 @@ export interface KBModelConfigRequest {
chunkSize: number
chunkOverlap: number
separators: string[]
parserEngineRules?: { file_types: string[]; engine: string }[]
}
multimodal: {
enabled: boolean
storageType?: 'cos' | 'minio'
cos?: {
secretId: string
secretKey: string
region: string
bucketName: string
appId: string
pathPrefix: string
}
minio?: {
bucketName: string
useSSL: boolean
pathPrefix: string
}
}
/** 存储引擎选择:"local" | "minio" | "cos",影响文档上传与文档内图片存储 */
storageProvider?: string
nodeExtract: {
enabled: boolean
text: string

View File

@ -106,6 +106,10 @@ export function downKnowledgeDetails(id: string) {
return getDown(`/api/v1/knowledge/${id}/download`);
}
export function previewKnowledgeFile(id: string) {
return getDown(`/api/v1/knowledge/${id}/preview`);
}
/** @param idsQueryString - query string with ids (e.g. ids=xxx&ids=yyy) */
export function batchQueryKnowledge(idsQueryString: string, kbId?: string, agentId?: string) {
let qs = idsQueryString;

View File

@ -1,7 +1,8 @@
import { get, put } from '@/utils/request'
import { get, post, put } from '@/utils/request'
export interface SystemInfo {
version: string
edition?: string
commit_id?: string
build_time?: string
go_version?: string
@ -110,3 +111,115 @@ export interface ListMinioBucketsResponse {
export function listMinioBuckets(): Promise<{ data: ListMinioBucketsResponse }> {
return get('/api/v1/system/minio/buckets')
}
export interface ParserEngineInfo {
Name: string
Description: string
FileTypes: string[]
Available?: boolean
UnavailableReason?: string
}
/** 解析引擎配置(引擎相关存租户;docreader 地址由环境变量配置) */
export interface ParserEngineConfig {
docreader_addr?: string
docreader_transport?: string
mineru_endpoint?: string
mineru_api_key?: string
mineru_api_base_url?: string
// MinerU 自建参数
mineru_model?: string
mineru_enable_formula?: boolean | null
mineru_enable_table?: boolean | null
mineru_enable_ocr?: boolean | null
mineru_language?: string
// MinerU 云 API 参数
mineru_cloud_model?: string
mineru_cloud_enable_formula?: boolean | null
mineru_cloud_enable_table?: boolean | null
mineru_cloud_enable_ocr?: boolean | null
mineru_cloud_language?: string
}
export interface ParserEnginesResponse {
data: ParserEngineInfo[]
docreader_addr?: string
/** 连接方式:grpc | http,由服务端环境/配置决定 */
docreader_transport?: string
connected?: boolean
}
export function getParserEngines(): Promise<ParserEnginesResponse> {
return get('/api/v1/system/parser-engines')
}
/** 使用当前填写的参数检测引擎可用性(不保存),用于填写新参数后即时测试 */
export function checkParserEngines(config: ParserEngineConfig): Promise<ParserEnginesResponse> {
return post('/api/v1/system/parser-engines/check', config)
}
export function getParserEngineConfig(): Promise<{ data: ParserEngineConfig }> {
return get('/api/v1/tenants/kv/parser-engine-config')
}
export function updateParserEngineConfig(config: ParserEngineConfig): Promise<{ data: ParserEngineConfig }> {
return put('/api/v1/tenants/kv/parser-engine-config', config)
}
export function reconnectDocReader(addr: string): Promise<ParserEnginesResponse & { msg?: string }> {
return post('/api/v1/system/docreader/reconnect', { addr })
}
// ---- 存储引擎配置(租户级,供文档/图片存储与 docreader 使用) ----
export interface StorageEngineConfig {
default_provider: string // "local" | "minio" | "cos"
local?: { path_prefix: string }
minio?: { mode: string; endpoint: string; access_key_id: string; secret_access_key: string; bucket_name: string; use_ssl: boolean; path_prefix: string }
cos?: {
secret_id: string
secret_key: string
region: string
bucket_name: string
app_id: string
path_prefix: string
}
}
export interface StorageEngineStatusItem {
name: string
available: boolean
description: string
}
export interface GetStorageEngineStatusResponse {
engines: StorageEngineStatusItem[]
minio_env_available: boolean
}
export function getStorageEngineConfig(): Promise<{ data: StorageEngineConfig }> {
return get('/api/v1/tenants/kv/storage-engine-config')
}
export function updateStorageEngineConfig(config: StorageEngineConfig): Promise<{ data: StorageEngineConfig }> {
return put('/api/v1/tenants/kv/storage-engine-config', config)
}
export function getStorageEngineStatus(): Promise<{ data: GetStorageEngineStatusResponse }> {
return get('/api/v1/system/storage-engine-status')
}
export interface StorageCheckRequest {
provider: string // "minio" | "cos"
minio?: StorageEngineConfig['minio']
cos?: StorageEngineConfig['cos']
}
export interface StorageCheckResponse {
ok: boolean
message: string
}
export function checkStorageEngine(req: StorageCheckRequest): Promise<{ data: StorageCheckResponse }> {
return post('/api/v1/system/storage-engine-check', req)
}

View File

@ -1,5 +1,5 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M10 3L10.8 6.2C10.9 6.7 11.3 7.1 11.8 7.2L15 8L11.8 8.8C11.3 8.9 10.9 9.3 10.8 9.8L10 13L9.2 9.8C9.1 9.3 8.7 8.9 8.2 8.8L5 8L8.2 7.2C8.7 7.1 9.1 6.7 9.2 6.2L10 3Z" fill="#000000" fill-opacity=".9" stroke="#000000" stroke-opacity=".9" stroke-width="0.8" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M15.5 4L15.8 5.2C15.85 5.45 16.05 5.65 16.3 5.7L17.5 6L16.3 6.3C16.05 6.35 15.85 6.55 15.8 6.8L15.5 8L15.2 6.8C15.15 6.55 14.95 6.35 14.7 6.3L13.5 6L14.7 5.7C14.95 5.65 15.15 5.45 15.2 5.2L15.5 4Z" fill="#000000" fill-opacity=".9" stroke="#000000" stroke-opacity=".9" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M4.5 13L4.8 14.2C4.85 14.45 5.05 14.65 5.3 14.7L6.5 15L5.3 15.3C5.05 15.35 4.85 15.55 4.8 15.8L4.5 17L4.2 15.8C4.15 15.55 3.95 15.35 3.7 15.3L2.5 15L3.7 14.7C3.95 14.65 4.15 14.45 4.2 14.2L4.5 13Z" fill="#000000" fill-opacity=".9" stroke="#000000" stroke-opacity=".9" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M10 3L10.8 6.2C10.9 6.7 11.3 7.1 11.8 7.2L15 8L11.8 8.8C11.3 8.9 10.9 9.3 10.8 9.8L10 13L9.2 9.8C9.1 9.3 8.7 8.9 8.2 8.8L5 8L8.2 7.2C8.7 7.1 9.1 6.7 9.2 6.2L10 3Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.8" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M15.5 4L15.8 5.2C15.85 5.45 16.05 5.65 16.3 5.7L17.5 6L16.3 6.3C16.05 6.35 15.85 6.55 15.8 6.8L15.5 8L15.2 6.8C15.15 6.55 14.95 6.35 14.7 6.3L13.5 6L14.7 5.7C14.95 5.65 15.15 5.45 15.2 5.2L15.5 4Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M4.5 13L4.8 14.2C4.85 14.45 5.05 14.65 5.3 14.7L6.5 15L5.3 15.3C5.05 15.35 4.85 15.55 4.8 15.8L4.5 17L4.2 15.8C4.15 15.55 3.95 15.35 3.7 15.3L2.5 15L3.7 14.7C3.95 14.65 4.15 14.45 4.2 14.2L4.5 13Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
</svg>

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

View File

@ -1,3 +1,3 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="#8A8A8A" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="#000000" stroke-opacity=".4" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
</svg>

Before

Width:  |  Height:  |  Size: 421 B

After

Width:  |  Height:  |  Size: 441 B

View File

@ -1,3 +1,3 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="#000000" stroke-opacity=".6" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
</svg>

Before

Width:  |  Height:  |  Size: 426 B

After

Width:  |  Height:  |  Size: 441 B

View File

@ -1,4 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="45o4l8wsy__design-iconfont" width="128" height="128">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#07C05F" fill-opacity=".9"></path>
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#07C05F" fill-opacity=".9"></path>
</svg>

Before

Width:  |  Height:  |  Size: 851 B

After

Width:  |  Height:  |  Size: 814 B

View File

@ -1,4 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="45o4l8wsy__design-iconfont" width="128" height="128">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#737373" fill-opacity=".9"></path>
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#737373" fill-opacity=".9"></path>
</svg>

Before

Width:  |  Height:  |  Size: 851 B

After

Width:  |  Height:  |  Size: 814 B

View File

@ -1,4 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="45o4l8wsy__design-iconfont" width="128" height="128">
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#000000" fill-opacity=".9"></path>
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#000000" fill-opacity=".9"></path>
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#000000" fill-opacity=".6"></path>
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#000000" fill-opacity=".6"></path>
</svg>

Before

Width:  |  Height:  |  Size: 851 B

After

Width:  |  Height:  |  Size: 814 B

View File

@ -1,3 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="cvw0hqhj9__design-iconfont" width="128" height="128">
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#07C05F"></path>
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#07C05F" stroke="#07C05F" stroke-width="0.3"/>
</svg>

Before

Width:  |  Height:  |  Size: 3.0 KiB

After

Width:  |  Height:  |  Size: 3.0 KiB

View File

@ -1,3 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="cvw0hqhj9__design-iconfont" width="128" height="128">
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#717171"></path>
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.3"/>
</svg>

Before

Width:  |  Height:  |  Size: 3.0 KiB

After

Width:  |  Height:  |  Size: 3.0 KiB

View File

@ -97,6 +97,9 @@ const agentKBSelectionMode = computed(() => {
return currentAgentConfig.value?.kb_selection_mode || 'all';
});
// 共享智能体下的知识库列表(来自 listKnowledgeBases(agent_id)),用于已选知识库展示与 org 角标
const sharedAgentKbList = ref<Array<{ id: string; name: string; type?: string; knowledge_count?: number; chunk_count?: number }>>([]);
// 当智能体改变时,模型、网络搜索、可@知识库列表均跟随新智能体配置
// 知识库:用新智能体配置的列表替换当前选中,使已选与可@列表一致(含共享智能体)
watch([selectedAgentId, agentKnowledgeBases, agentKBSelectionMode], ([newAgentId, newAgentKbs, newKbMode], [oldAgentId]) => {
@ -205,9 +208,6 @@ const sharedAgentOrgName = computed(() => {
return shared?.org_name || shared?.shared_by_username || '';
});
// 共享智能体下的知识库列表(来自 listKnowledgeBases(agent_id)),用于已选知识库展示与 org 角标
const sharedAgentKbList = ref<Array<{ id: string; name: string; type?: string; knowledge_count?: number; chunk_count?: number }>>([]);
const props = defineProps({
isReplying: {
type: Boolean,

View File

@ -10,6 +10,7 @@ import { MessagePlugin, DialogPlugin } from "tdesign-vue-next";
import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL } from '@/utils/security';
import { openMermaidFullscreen } from '@/utils/mermaidViewer';
import { useI18n } from 'vue-i18n';
import DocumentPreview from '@/components/document-preview.vue';
const { t } = useI18n();
@ -57,10 +58,9 @@ let doc = null;
let down = ref()
let mdContentWrap = ref()
let url = ref('')
// 视图模式:chunks / original / merged
const viewMode = ref<'chunks' | 'original' | 'merged'>('merged');
const originalContent = ref<string>('');
const loadingOriginal = ref(false);
// 视图模式:chunks / merged / preview
// file 类型默认「预览」,URL / 手动创建 默认「全文」
const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged');
// 合并后的文档内容
const mergedContent = ref<string>('');
@ -212,6 +212,31 @@ watch(() => props.details?.md, (newChunks) => {
}
}, { immediate: true, deep: true });
const previewSupportedTypes = new Set([
'pdf', 'docx', 'pptx', 'ppt', 'xlsx', 'xls', 'csv',
'jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'tiff', 'svg',
'txt', 'md', 'markdown', 'json', 'xml', 'html', 'css', 'js', 'ts',
'py', 'java', 'go', 'cpp', 'c', 'h', 'sh', 'yaml', 'yml',
'ini', 'conf', 'log', 'sql', 'rs', 'rb', 'php', 'swift', 'kt',
'scala', 'r', 'lua', 'pl', 'toml',
]);
const canPreview = (): boolean => {
if (props.details?.type !== 'file') return false;
const ft = props.details?.file_type?.toLowerCase();
return !!ft && previewSupportedTypes.has(ft);
};
// 当文档详情加载完成时,file 类型自动切换到「预览」
watch(() => props.details?.id, (newId) => {
if (!newId) return;
if (props.details?.type === 'file' && canPreview()) {
viewMode.value = 'preview';
} else {
viewMode.value = 'merged';
}
});
const isTextFile = (fileType?: string): boolean => {
if (!fileType) return false;
const textTypes = ['txt', 'md', 'markdown', 'json', 'xml', 'html', 'css', 'js', 'ts', 'py', 'java', 'go', 'cpp', 'c', 'h', 'sh', 'yaml', 'yml', 'ini', 'conf', 'log'];
@ -222,25 +247,6 @@ const isMarkdownFile = (fileType?: string): boolean => {
const markdownTypes = ['md', 'markdown'];
return markdownTypes.includes(fileType.toLowerCase());
};
const loadOriginalContent = async () => {
if (!props.details.id || !props.details.type || props.details.type !== 'file') return;
const fileType = props.details.file_type?.toLowerCase();
if (!isTextFile(fileType)) {
MessagePlugin.warning(t('knowledgeBase.originalFileNotSupported') || '该文件类型不支持原文件展示,请下载查看');
return;
}
loadingOriginal.value = true;
try {
const blob = await downKnowledgeDetails(props.details.id);
const text = await blob.text();
originalContent.value = text;
} catch (error: any) {
console.error('Failed to load original content:', error);
MessagePlugin.error(error?.message || t('knowledgeBase.loadOriginalFailed') || '加载原文件内容失败');
} finally {
loadingOriginal.value = false;
}
};
watch(() => props.details.md, (newVal) => {
nextTick(async () => {
const images = mdContentWrap.value.querySelectorAll('img.markdown-image');
@ -346,7 +352,6 @@ const handleClose = () => {
emit("closeDoc", false);
doc.scrollTop = 0;
viewMode.value = 'merged';
originalContent.value = '';
};
// 获取显示标题
@ -622,6 +627,17 @@ const handleDetailsScroll = () => {
<span class="time"> {{ getTimeLabel() }}:{{ details.time }} </span>
<div class="view-mode-buttons">
<t-button
v-if="canPreview()"
size="small"
:variant="viewMode === 'preview' ? 'base' : 'outline'"
:theme="viewMode === 'preview' ? 'primary' : 'default'"
@click="viewMode = 'preview'"
class="view-mode-btn"
>
{{ $t('preview.tab') || '预览' }}
</t-button>
<t-button
v-if="!canPreview()"
size="small"
:variant="viewMode === 'merged' ? 'base' : 'outline'"
:theme="viewMode === 'merged' ? 'primary' : 'default'"
@ -639,7 +655,6 @@ const handleDetailsScroll = () => {
>
{{ $t('knowledgeBase.viewChunks') || '分块' }}
</t-button>
</div>
</div>
</div>
@ -709,6 +724,16 @@ const handleDetailsScroll = () => {
</div>
</div>
<!-- 文档预览视图 -->
<div v-else-if="viewMode === 'preview'">
<DocumentPreview
:knowledgeId="details.id"
:fileType="details.file_type"
:fileName="details.title"
:active="viewMode === 'preview'"
/>
</div>
<template #footer>
<t-button @click="handleClose">{{ $t('common.confirm') }}</t-button>
<t-button theme="default" @click="handleClose">{{ $t('common.cancel') }}</t-button>

View File

@ -0,0 +1,625 @@
// @ts-nocheck
<script setup lang="ts">
import { ref, shallowRef, watch, onUnmounted, nextTick, defineAsyncComponent } from 'vue';
import { previewKnowledgeFile } from '@/api/knowledge-base/index';
import { MessagePlugin } from 'tdesign-vue-next';
import hljs from 'highlight.js';
import 'highlight.js/styles/github.css';
import { useI18n } from 'vue-i18n';
const VueOfficePptx = defineAsyncComponent(() => import('@vue-office/pptx'));
const { t } = useI18n();
const props = defineProps<{
knowledgeId: string;
fileType: string;
fileName: string;
active: boolean;
}>();
const loading = ref(false);
const error = ref('');
const previewType = ref<'pdf' | 'docx' | 'image' | 'excel' | 'text' | 'markdown' | 'pptx' | 'unsupported'>('unsupported');
const blobUrl = ref('');
const textContent = ref('');
const highlightedCode = ref('');
const markdownHtml = ref('');
const excelHtml = ref('');
const pptxData = shallowRef<ArrayBuffer | null>(null);
const docxContainer = ref<HTMLElement | null>(null);
const imageNaturalWidth = ref(0);
const imageNaturalHeight = ref(0);
let loadedForId = '';
const fileTypeMap: Record<string, typeof previewType.value> = {};
['pdf'].forEach(t => fileTypeMap[t] = 'pdf');
['docx'].forEach(t => fileTypeMap[t] = 'docx');
['pptx', 'ppt'].forEach(t => fileTypeMap[t] = 'pptx');
['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'tiff', 'svg'].forEach(t => fileTypeMap[t] = 'image');
['xlsx', 'xls', 'csv'].forEach(t => fileTypeMap[t] = 'excel');
['md', 'markdown'].forEach(t => fileTypeMap[t] = 'markdown');
['txt', 'json', 'xml', 'html', 'css', 'js', 'ts', 'py', 'java', 'go',
'cpp', 'c', 'h', 'sh', 'yaml', 'yml', 'ini', 'conf', 'log', 'sql', 'rs', 'rb', 'php',
'swift', 'kt', 'scala', 'r', 'lua', 'pl', 'toml'].forEach(t => fileTypeMap[t] = 'text');
const mimeTypeMap: Record<string, string> = {
pdf: 'application/pdf',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
doc: 'application/msword',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
ppt: 'application/vnd.ms-powerpoint',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
xls: 'application/vnd.ms-excel',
csv: 'text/csv',
jpg: 'image/jpeg', jpeg: 'image/jpeg',
png: 'image/png', gif: 'image/gif', bmp: 'image/bmp',
webp: 'image/webp', tiff: 'image/tiff', svg: 'image/svg+xml',
txt: 'text/plain', md: 'text/markdown', markdown: 'text/markdown',
json: 'application/json', xml: 'application/xml',
html: 'text/html', css: 'text/css',
js: 'text/javascript', ts: 'text/typescript',
py: 'text/x-python', java: 'text/x-java', go: 'text/x-go',
};
function getMimeType(ft: string): string {
return mimeTypeMap[ft?.toLowerCase()] || 'application/octet-stream';
}
function ensureBlobType(blob: Blob, ft: string): Blob {
const expected = getMimeType(ft);
if (blob.type === expected) return blob;
return new Blob([blob], { type: expected });
}
const langMap: Record<string, string> = {
js: 'javascript', ts: 'typescript', py: 'python', rb: 'ruby',
sh: 'bash', yml: 'yaml', md: 'markdown', rs: 'rust',
kt: 'kotlin', pl: 'perl', conf: 'ini', log: 'plaintext',
};
function resolvePreviewType(ft: string): typeof previewType.value {
return fileTypeMap[ft?.toLowerCase()] || 'unsupported';
}
function getHighlightLang(ft: string): string {
const lower = ft?.toLowerCase() || '';
return langMap[lower] || lower;
}
async function renderDocx(blob: Blob) {
const { renderAsync } = await import('docx-preview');
if (docxContainer.value) {
docxContainer.value.innerHTML = '';
await renderAsync(blob, docxContainer.value, undefined, {
className: 'docx-preview-wrapper',
inWrapper: true,
ignoreWidth: false,
ignoreHeight: false,
ignoreFonts: false,
breakPages: true,
ignoreLastRenderedPageBreak: true,
experimental: false,
trimXmlDeclaration: true,
useBase64URL: true,
});
}
}
function isValidUTF8(bytes: Uint8Array): boolean {
for (let i = 0; i < bytes.length;) {
const b = bytes[i];
let remaining = 0;
if (b <= 0x7F) { remaining = 0; }
else if ((b & 0xE0) === 0xC0) { remaining = 1; }
else if ((b & 0xF0) === 0xE0) { remaining = 2; }
else if ((b & 0xF8) === 0xF0) { remaining = 3; }
else { return false; }
if (i + remaining >= bytes.length) return false;
for (let j = 1; j <= remaining; j++) {
if ((bytes[i + j] & 0xC0) !== 0x80) return false;
}
i += 1 + remaining;
}
return true;
}
function decodeCSVBlob(arrayBuffer: ArrayBuffer): string {
const bytes = new Uint8Array(arrayBuffer);
if (bytes[0] === 0xEF && bytes[1] === 0xBB && bytes[2] === 0xBF) {
return new TextDecoder('utf-8').decode(bytes);
}
if (isValidUTF8(bytes)) {
return new TextDecoder('utf-8').decode(bytes);
}
return new TextDecoder('gbk').decode(bytes);
}
async function renderExcel(blob: Blob, fileType?: string) {
const XLSX = await import('xlsx');
const arrayBuffer = await blob.arrayBuffer();
let workbook;
if (fileType?.toLowerCase() === 'csv') {
const csvText = decodeCSVBlob(arrayBuffer);
workbook = XLSX.read(csvText, { type: 'string' });
} else {
workbook = XLSX.read(arrayBuffer, { type: 'array' });
}
let html = '';
workbook.SheetNames.forEach((name, sheetIdx) => {
const sheet = workbook.Sheets[name];
const sheetHtml = XLSX.utils.sheet_to_html(sheet, { id: `sheet-${sheetIdx}` });
html += `<div class="excel-sheet">`;
if (workbook.SheetNames.length > 1) {
html += `<div class="excel-sheet-name">${name}</div>`;
}
html += sheetHtml;
html += `</div>`;
});
excelHtml.value = html;
}
async function renderText(blob: Blob, fileType: string) {
const text = await blob.text();
textContent.value = text;
const lang = getHighlightLang(fileType);
if (lang && hljs.getLanguage(lang)) {
try {
highlightedCode.value = hljs.highlight(text, { language: lang }).value;
return;
} catch { /* fallthrough */ }
}
const auto = hljs.highlightAuto(text);
highlightedCode.value = auto.value;
}
async function renderMarkdown(blob: Blob) {
const { marked } = await import('marked');
const text = await blob.text();
marked.use({
breaks: true,
gfm: true,
});
const renderer = new marked.Renderer();
renderer.code = function (code, infostring) {
const lang = (infostring || '').trim();
let highlighted = '';
if (lang && hljs.getLanguage(lang)) {
try { highlighted = hljs.highlight(code, { language: lang }).value; }
catch { highlighted = hljs.highlightAuto(code).value; }
} else {
highlighted = hljs.highlightAuto(code).value;
}
return `<pre><code class="hljs">${highlighted}</code></pre>`;
};
marked.use({ renderer });
markdownHtml.value = marked.parse(text);
}
function onImageLoad(e: Event) {
const img = e.target as HTMLImageElement;
imageNaturalWidth.value = img.naturalWidth;
imageNaturalHeight.value = img.naturalHeight;
}
async function loadPreview() {
const id = props.knowledgeId;
const ft = props.fileType;
if (!id || !ft) return;
if (loadedForId === id) return;
cleanup();
loading.value = true;
error.value = '';
previewType.value = resolvePreviewType(ft);
if (previewType.value === 'unsupported') {
loading.value = false;
return;
}
try {
const rawBlob = await previewKnowledgeFile(id);
const blob = ensureBlobType(rawBlob, ft);
loadedForId = id;
loading.value = false;
await nextTick();
switch (previewType.value) {
case 'pdf': {
blobUrl.value = URL.createObjectURL(blob);
break;
}
case 'image': {
blobUrl.value = URL.createObjectURL(blob);
break;
}
case 'docx': {
await renderDocx(blob);
break;
}
case 'excel': {
await renderExcel(blob, ft);
break;
}
case 'text': {
await renderText(blob, ft);
break;
}
case 'markdown': {
await renderMarkdown(blob);
break;
}
case 'pptx': {
pptxData.value = await blob.arrayBuffer();
break;
}
}
} catch (err: any) {
console.error('Document preview failed:', err);
error.value = err?.message || t('preview.loadFailed');
} finally {
loading.value = false;
}
}
function cleanup() {
if (blobUrl.value) {
URL.revokeObjectURL(blobUrl.value);
blobUrl.value = '';
}
textContent.value = '';
highlightedCode.value = '';
markdownHtml.value = '';
excelHtml.value = '';
pptxData.value = null;
imageNaturalWidth.value = 0;
imageNaturalHeight.value = 0;
loadedForId = '';
if (docxContainer.value) {
docxContainer.value.innerHTML = '';
}
}
watch(
() => [props.active, props.knowledgeId],
([active]) => {
if (active && props.knowledgeId) {
loadPreview();
}
},
{ immediate: true }
);
onUnmounted(() => {
cleanup();
});
</script>
<template>
<div class="document-preview">
<!-- Loading -->
<div v-if="loading" class="preview-loading">
<t-loading size="large" />
<span class="loading-text">{{ $t('preview.loading') }}</span>
</div>
<!-- Error -->
<div v-else-if="error" class="preview-error">
<t-icon name="error-circle" size="48px" />
<p>{{ error }}</p>
<t-button theme="primary" size="small" @click="loadedForId = ''; loadPreview()">
{{ $t('preview.retry') }}
</t-button>
</div>
<!-- Unsupported -->
<div v-else-if="previewType === 'unsupported'" class="preview-unsupported">
<t-icon name="file-unknown" size="48px" />
<p>{{ $t('preview.unsupported') }}</p>
<p class="unsupported-hint">{{ $t('preview.unsupportedHint') }}</p>
</div>
<!-- PDF -->
<div v-else-if="previewType === 'pdf' && blobUrl" class="preview-pdf">
<iframe :src="blobUrl" class="pdf-iframe" />
</div>
<!-- Image -->
<div v-else-if="previewType === 'image' && blobUrl" class="preview-image">
<div class="image-wrapper">
<img :src="blobUrl" :alt="fileName" @load="onImageLoad" />
<div v-if="imageNaturalWidth" class="image-info">
{{ imageNaturalWidth }} × {{ imageNaturalHeight }} px
</div>
</div>
</div>
<!-- DOCX -->
<div v-else-if="previewType === 'docx'" class="preview-docx">
<div ref="docxContainer" class="docx-container" />
</div>
<!-- PPTX -->
<div v-else-if="previewType === 'pptx' && pptxData" class="preview-pptx">
<vue-office-pptx :src="pptxData" @rendered="() => {}" @error="(e: any) => { error = e?.message || $t('preview.loadFailed'); }" />
</div>
<!-- Excel -->
<div v-else-if="previewType === 'excel' && excelHtml" class="preview-excel">
<div class="excel-container" v-html="excelHtml" />
</div>
<!-- Markdown -->
<div v-else-if="previewType === 'markdown' && markdownHtml" class="preview-markdown">
<div class="markdown-body" v-html="markdownHtml" />
</div>
<!-- Text / Code -->
<div v-else-if="previewType === 'text' && highlightedCode" class="preview-text">
<pre class="code-preview"><code class="hljs" v-html="highlightedCode"></code></pre>
</div>
</div>
</template>
<style scoped lang="less">
// ── Design tokens ──
@border-color: #e7e7e7;
@border-radius: 6px;
@bg-white: #fff;
@bg-subtle: #f6f8fa;
@bg-muted: #f9fafb;
@text-primary: #1d2129;
@text-secondary: #000000a6;
@text-tertiary: #00000066;
@text-disabled: #00000040;
@accent: #07c05f;
@accent-hover: #059669;
@accent-bg: #f0fdf4;
@accent-bg-hover: #e6f7ed;
@error-color: #e34d59;
@table-border: #e7e7e7;
@preview-max-h: calc(100vh - 200px);
@transition: all 0.2s ease;
// ── Shared container mixin ──
.preview-container() {
border: 1px solid @border-color;
border-radius: @border-radius;
overflow: auto;
max-height: @preview-max-h;
background: @bg-white;
}
.document-preview {
min-height: 200px;
}
// ── States ──
.preview-loading {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 60px 20px;
gap: 16px;
.loading-text { color: @text-tertiary; font-size: 14px; }
}
.preview-error {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 60px 20px;
gap: 12px;
color: @error-color;
p { margin: 0; font-size: 14px; color: @text-secondary; }
}
.preview-unsupported {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 60px 20px;
gap: 12px;
color: @text-disabled;
p { margin: 0; font-size: 14px; color: @text-secondary; }
.unsupported-hint { font-size: 12px; color: @text-tertiary; }
}
// ── PDF ──
.preview-pdf {
width: 100%;
height: @preview-max-h;
min-height: 500px;
.pdf-iframe {
width: 100%;
height: 100%;
border: none;
border-radius: @border-radius;
}
}
// ── Image ──
.preview-image {
display: flex;
justify-content: center;
padding: 20px 0;
.image-wrapper {
display: flex;
flex-direction: column;
align-items: center;
gap: 8px;
img {
max-width: 100%;
max-height: calc(100vh - 280px);
border-radius: @border-radius;
box-shadow: 0 2px 12px rgba(7, 192, 95, 0.08);
object-fit: contain;
}
.image-info { font-size: 12px; color: @text-tertiary; }
}
}
// ── Markdown ──
.preview-markdown {
.preview-container();
padding: 20px 24px;
}
// ── DOCX ──
.preview-docx {
.docx-container { .preview-container(); }
}
// ── PPTX ──
.preview-pptx {
height: @preview-max-h;
min-height: 500px;
border: 1px solid @border-color;
border-radius: @border-radius;
overflow: hidden;
background: @bg-subtle;
}
// ── Excel ──
.preview-excel {
.excel-container { .preview-container(); }
}
// ── Text / Code ──
.preview-text {
.code-preview {
.preview-container();
margin: 0;
padding: 16px;
background: @bg-subtle;
font-size: 13px;
line-height: 1.6;
code {
white-space: pre;
word-wrap: normal;
display: block;
background: transparent;
}
}
}
// ── Deep styles (v-html / third-party components) ──
// Shared table mixin for v-html content
.preview-table() {
width: 100%;
border-collapse: collapse;
font-size: 13px;
th, td {
border: 1px solid @table-border;
padding: 6px 12px;
text-align: left;
}
th {
background: @accent-bg;
font-weight: 600;
color: @text-primary;
}
tr:hover td {
background: @accent-bg;
transition: @transition;
}
}
:deep(.markdown-body) {
font-size: 14px;
line-height: 1.7;
color: @text-primary;
word-break: break-word;
h1, h2, h3, h4, h5, h6 {
margin-top: 20px;
margin-bottom: 10px;
font-weight: 600;
line-height: 1.4;
}
h1 { font-size: 24px; border-bottom: 1px solid @border-color; padding-bottom: 8px; }
h2 { font-size: 20px; border-bottom: 1px solid @border-color; padding-bottom: 6px; }
h3 { font-size: 17px; }
p { margin: 8px 0; }
blockquote {
margin: 12px 0;
padding: 8px 16px;
border-left: 4px solid @accent;
background: @bg-subtle;
color: #57606a;
}
ul, ol { padding-left: 24px; margin: 8px 0; }
li { margin: 4px 0; }
table { .preview-table(); margin: 12px 0; }
pre {
margin: 12px 0;
padding: 14px;
background: @bg-subtle;
border-radius: @border-radius;
overflow: auto;
font-size: 13px;
line-height: 1.5;
code { background: transparent; padding: 0; }
}
code {
background: #eff1f3;
padding: 2px 6px;
border-radius: 3px;
font-size: 0.9em;
}
img { max-width: 100%; border-radius: 4px; }
hr { border: none; border-top: 1px solid @border-color; margin: 20px 0; }
a { color: @accent; text-decoration: none; &:hover { color: @accent-hover; text-decoration: underline; } }
strong { font-weight: 600; }
}
:deep(.docx-preview-wrapper) {
padding: 20px;
}
:deep(.vue-office-pptx) {
width: 100%;
height: 100%;
}
:deep(.vue-office-pptx-main) {
width: 100%;
height: 100%;
}
:deep(.excel-sheet) {
padding: 0;
.excel-sheet-name {
position: sticky;
top: 0;
background: @accent-bg;
padding: 8px 16px;
font-weight: 600;
font-size: 13px;
color: @text-primary;
border-bottom: 1px solid @border-color;
z-index: 1;
}
table {
.preview-table();
th, td {
white-space: nowrap;
max-width: 300px;
overflow: hidden;
text-overflow: ellipsis;
}
}
}
</style>

View File

@ -2,6 +2,7 @@
<div class="aside_box">
<div class="logo_box" @click="router.push('/platform/knowledge-bases')" style="cursor: pointer;">
<img class="logo" src="@/assets/img/weknora.png" alt="">
<span v-if="isLiteEdition" class="lite-badge">Lite</span>
</div>
<!-- 租户选择器:仅在用户可切换租户时显示 -->
@ -80,6 +81,7 @@ import UserMenu from '@/components/UserMenu.vue';
import TenantSelector from '@/components/TenantSelector.vue';
import BatchManageDialog from '@/components/BatchManageDialog.vue';
import { useI18n } from 'vue-i18n';
import { getSystemInfo } from '@/api/system';
const { t } = useI18n();
const usemenuStore = useMenuStore();
@ -100,6 +102,7 @@ const hasMore = computed(() => currentPage.value < totalPages.value);
type MenuItem = { title: string; icon: string; path: string; childrenPath?: string; children?: any[] };
const { menuArr } = storeToRefs(usemenuStore);
let activeSubmenu = ref<string>('');
const isLiteEdition = ref(false);
// 批量管理状态
const batchManageVisible = ref(false);
@ -380,6 +383,10 @@ onMounted(async () => {
if (route.params.chatid) {
currentSecondpath.value = `chat/${route.params.chatid}`;
}
getSystemInfo().then(res => {
isLiteEdition.value = res.data?.edition === 'lite'
}).catch(() => {})
// 初始化知识库信息
const kbId = (route.params as any)?.kbId as string
@ -593,6 +600,19 @@ const mouseleaveMenu = (path: string) => {
height: auto;
margin-left: 24px;
}
.lite-badge {
margin-left: 4px;
padding: 1px 6px;
font-size: 11px;
font-weight: 600;
line-height: 18px;
color: #07c05f;
background: #07c05f1a;
border: 1px solid #07c05f40;
border-radius: 4px;
letter-spacing: 0.5px;
user-select: none;
}
}
.logo_img {
@ -699,6 +719,8 @@ const mouseleaveMenu = (path: string) => {
padding: 13px 8px 13px 16px;
box-sizing: border-box;
margin-bottom: 4px;
border-radius: 4px;
transition: background-color 0.2s ease;
.menu_item-box {
display: flex;
@ -707,12 +729,11 @@ const mouseleaveMenu = (path: string) => {
&:hover {
border-radius: 4px;
background: #30323605;
color: #00000099;
background: rgba(0, 0, 0, 0.04);
.menu_icon,
.menu_title {
color: #00000099;
color: #000000b3;
}
}
}
@ -812,12 +833,12 @@ const mouseleaveMenu = (path: string) => {
}
&:hover {
background: #30323605;
color: #00000099;
background: rgba(0, 0, 0, 0.04);
color: #000000b3;
border-radius: 3px;
.menu-more {
color: #00000099;
color: #000000b3;
}
.menu-more-wrap {

View File

@ -97,6 +97,8 @@ export default {
imagesFilteredNoVLM: 'Filtered {count} image files (VLM not enabled)',
invalidFilesFiltered: 'Filtered {count} unsupported files',
unsupportedFileType: 'Unsupported file type',
unsupportedTypesHint: 'Some document types ({types}) have no available parser engine and cannot be processed',
goToParserSettings: 'Configure',
failedFilesList: 'Failed files:',
andMoreFiles: '...and {count} more files',
duplicateFilesSkipped: '{count} duplicate files skipped',
@ -504,6 +506,7 @@ export default {
webSearchConfig: 'Web Search',
enableMemory: 'Enable Memory',
enableMemoryDesc: 'When enabled, the system will record your conversation history and automatically recall relevant content in future conversations to provide more personalized answers.',
storageEngine: 'Storage Engine',
mcpService: 'MCP Service',
systemSettings: 'System Settings',
tenantInfo: 'Tenant Info',
@ -1216,6 +1219,7 @@ export default {
basic: 'Basic Information',
models: 'Model Configuration',
chunking: 'Chunking Settings',
storage: 'Storage Engine',
advanced: 'Advanced Settings',
faq: 'FAQ Settings',
graph: 'Knowledge Graph',
@ -1503,6 +1507,11 @@ export default {
deepThoughtCompleted: 'Deep thinking completed',
deepThoughtAlt: 'Deep thinking finished',
referencesTitle: 'Referenced {count} related item(s)',
referencesDocCount: 'Referenced {count} document(s)',
referencesDocAndWebCount: 'Referenced {docCount} document(s) and {webCount} web page(s)',
referenceChunkCount: '{count} chunk(s)',
chunkLabel: 'Chunk {index}:',
navigateToDocument: 'View document details',
referenceIconAlt: 'Reference materials icon',
chunkIdLabel: 'Chunk ID:',
documentIdLabel: 'Document ID:',
@ -2698,4 +2707,12 @@ export default {
sharedAgents: ' agents',
},
},
preview: {
tab: 'Preview',
loading: 'Loading document preview...',
loadFailed: 'Failed to load document preview',
retry: 'Retry',
unsupported: 'This file type does not support online preview',
unsupportedHint: 'Please download and open with a local application',
},
}

View File

@ -96,6 +96,8 @@ export default {
imagesFilteredNoVLM: "{count}개 이미지 파일 필터링됨(VLM 미활성화)",
invalidFilesFiltered: "{count}개 지원되지 않는 파일 필터링됨",
unsupportedFileType: "지원되지 않는 파일 형식",
unsupportedTypesHint: "일부 문서 유형({types})에 사용 가능한 파서 엔진이 없어 처리할 수 없습니다",
goToParserSettings: "설정으로 이동",
failedFilesList: "실패한 파일 목록:",
andMoreFiles: "...외 {count}개 파일",
duplicateFilesSkipped: "{count}개 중복 파일 무시됨",
@ -273,6 +275,11 @@ export default {
deepThoughtCompleted: "심층 분석 완료",
deepThoughtAlt: "심층 분석 완료",
referencesTitle: "{count}개의 관련 내용 참조",
referencesDocCount: "{count}개 문서 참조",
referencesDocAndWebCount: "{docCount}개 문서와 {webCount}개 웹페이지 참조",
referenceChunkCount: "{count}개 청크",
chunkLabel: "청크 {index}:",
navigateToDocument: "문서 상세 보기",
referenceIconAlt: "참조 내용 아이콘",
chunkIdLabel: "청크 ID:",
documentIdLabel: "문서 ID:",
@ -2777,4 +2784,12 @@ export default {
sharedAgents: "에이전트",
},
},
preview: {
tab: '미리보기',
loading: '문서 미리보기 로딩 중...',
loadFailed: '문서 미리보기 로드 실패',
retry: '재시도',
unsupported: '이 파일 유형은 온라인 미리보기를 지원하지 않습니다',
unsupportedHint: '파일을 다운로드하여 로컬 앱으로 열어주세요',
},
};

View File

@ -77,6 +77,8 @@ export default {
imagesFilteredNoVLM: 'Отфильтровано {count} изображений (VLM не включен)',
invalidFilesFiltered: 'Отфильтровано {count} неподдерживаемых файлов',
unsupportedFileType: 'Неподдерживаемый тип файла',
unsupportedTypesHint: 'Некоторые типы документов ({types}) не имеют доступного парсера и не могут быть обработаны',
goToParserSettings: 'Настроить',
failedFilesList: 'Неудавшиеся файлы:',
andMoreFiles: '...и ещё {count} файлов',
duplicateFilesSkipped: 'Пропущено {count} повторяющихся файлов',
@ -1331,6 +1333,11 @@ export default {
unableToGetKnowledgeBaseId: 'Невозможно получить ID базы знаний',
summaryInProgress: 'Идёт подготовка ответа…',
referencesTitle: 'Использовано {count} связанного материала',
referencesDocCount: 'Использовано {count} документ(ов)',
referencesDocAndWebCount: 'Использовано {docCount} документ(ов) и {webCount} веб-страниц(ы)',
referenceChunkCount: '{count} фрагмент(ов)',
chunkLabel: 'Фрагмент {index}:',
navigateToDocument: 'Просмотр документа',
referenceIconAlt: 'Иконка ссылок на материалы',
chunkIdLabel: 'ID фрагмента:',
documentIdLabel: 'ID документа:',
@ -1841,5 +1848,13 @@ export default {
stopSuccess: 'Генерация остановлена',
stopFailed: 'Не удалось остановить. Попробуйте ещё раз.'
}
}
},
preview: {
tab: 'Предпросмотр',
loading: 'Загрузка предпросмотра документа...',
loadFailed: 'Не удалось загрузить предпросмотр документа',
retry: 'Повторить',
unsupported: 'Этот тип файла не поддерживает онлайн-просмотр',
unsupportedHint: 'Скачайте файл и откройте локально',
},
}

View File

@ -96,6 +96,8 @@ export default {
imagesFilteredNoVLM: "已过滤 {count} 个图片文件(未启用VLM)",
invalidFilesFiltered: "已过滤 {count} 个不支持的文件",
unsupportedFileType: "不支持的文件格式",
unsupportedTypesHint: "部分文档类型({types})暂无可用解析引擎,上传后将无法解析",
goToParserSettings: "前往配置",
failedFilesList: "失败文件列表:",
andMoreFiles: "...及其他 {count} 个文件",
duplicateFilesSkipped: "已忽略 {count} 个重复文件",
@ -270,6 +272,11 @@ export default {
deepThoughtCompleted: "已深度思考",
deepThoughtAlt: "深度思考完成",
referencesTitle: "参考了{count}个相关内容",
referencesDocCount: "引用了{count}篇文档",
referencesDocAndWebCount: "引用了{docCount}篇文档和{webCount}条网页",
referenceChunkCount: "{count}个片段",
chunkLabel: "片段{index}:",
navigateToDocument: "查看文档详情",
referenceIconAlt: "参考内容图标",
chunkIdLabel: "片段ID:",
documentIdLabel: "文档ID:",
@ -346,6 +353,7 @@ export default {
webSearchConfig: "网络搜索",
enableMemory: "开启记忆功能",
enableMemoryDesc: "开启后,系统将记录您的对话历史,并在后续对话中自动回忆相关内容,提供更个性化的回答。",
storageEngine: "存储引擎",
mcpService: "MCP服务",
systemSettings: "系统设置",
tenantInfo: "租户信息",
@ -1612,6 +1620,7 @@ export default {
basic: "基本信息",
models: "模型配置",
chunking: "分块设置",
storage: "存储引擎",
advanced: "高级设置",
faq: "FAQ 设置",
graph: "知识图谱",
@ -2703,4 +2712,12 @@ export default {
sharedAgents: "个智能体",
},
},
preview: {
tab: "预览",
loading: "正在加载文档预览...",
loadFailed: "加载文档预览失败",
retry: "重试",
unsupported: "该文件类型暂不支持在线预览",
unsupportedHint: "请下载文件后使用本地应用查看",
},
};

View File

@ -39,29 +39,29 @@ export function formatStringDate(date: any) {
year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second
);
}
export function kbFileTypeVerification(file: any, silent = false) {
let validTypes = ["pdf", "txt", "md", "docx", "doc", "jpg", "jpeg", "png", "csv", "xlsx", "xls"];
let type = file.name.substring(file.name.lastIndexOf(".") + 1);
if (!validTypes.includes(type)) {
const DEFAULT_VALID_TYPES = new Set(["pdf", "txt", "md", "docx", "doc", "pptx", "ppt", "jpg", "jpeg", "png", "csv", "xlsx", "xls"]);
/**
* Returns true when the file should be **rejected**.
* @param validTypes - override the default extension whitelist with a dynamic set (e.g. from engine registry).
*/
export function kbFileTypeVerification(file: any, silent = false, validTypes?: Set<string> | string[]) {
const allowed = validTypes
? (validTypes instanceof Set ? validTypes : new Set(validTypes))
: DEFAULT_VALID_TYPES;
const type = file.name.substring(file.name.lastIndexOf(".") + 1).toLowerCase();
if (!allowed.has(type)) {
if (!silent) {
MessagePlugin.error("文件类型错误!");
MessagePlugin.error("不支持的文件类型!");
}
return true;
}
if (
(type == "pdf" || type == "docx" || type == "doc") &&
file.size > MAX_FILE_SIZE_BYTES
) {
if (file.size > MAX_FILE_SIZE_BYTES) {
if (!silent) {
MessagePlugin.error(`pdf/doc文件不能超过${MAX_FILE_SIZE_MB}M!`);
MessagePlugin.error(`文件大小不能超过 ${MAX_FILE_SIZE_MB}M!`);
}
return true;
}
if ((type == "txt" || type == "md") && file.size > MAX_FILE_SIZE_BYTES) {
if (!silent) {
MessagePlugin.error(`txt/md文件不能超过${MAX_FILE_SIZE_MB}M!`);
}
return true;
}
return false
return false;
}

View File

@ -140,10 +140,14 @@ export function isValidURL(url: string): boolean {
if (!url || typeof url !== 'string') {
return false;
}
// 允许以 / 开头的站内相对路径(如本地存储 /files/images/xxx.jpg)
if (url.startsWith('/') && !url.startsWith('//')) {
return true;
}
try {
const urlObj = new URL(url);
// 只允许 http 和 https 协议
return ['http:', 'https:'].includes(urlObj.protocol);
} catch {
return false;

View File

@ -890,7 +890,7 @@
</div>
<div class="setting-control">
<div class="slider-wrapper">
<t-slider v-model="formData.config.keyword_threshold" :min="0" :max="1" :step="0.05" />
<t-slider v-model="formData.config.keyword_threshold" :min="0" :max="1" :step="0.01" />
<span class="slider-value">{{ formData.config.keyword_threshold?.toFixed(2) }}</span>
</div>
</div>
@ -904,7 +904,7 @@
</div>
<div class="setting-control">
<div class="slider-wrapper">
<t-slider v-model="formData.config.vector_threshold" :min="0" :max="1" :step="0.05" />
<t-slider v-model="formData.config.vector_threshold" :min="0" :max="1" :step="0.01" />
<span class="slider-value">{{ formData.config.vector_threshold?.toFixed(2) }}</span>
</div>
</div>
@ -929,7 +929,7 @@
</div>
<div class="setting-control">
<div class="slider-wrapper">
<t-slider v-model="formData.config.rerank_threshold" :min="0" :max="1" :step="0.05" />
<t-slider v-model="formData.config.rerank_threshold" :min="0" :max="1" :step="0.01" />
<span class="slider-value">{{ formData.config.rerank_threshold?.toFixed(2) }}</span>
</div>
</div>

View File

@ -3,48 +3,71 @@
<div class="refer_header" @click="referBoxSwitch" v-if="session.knowledge_references && session.knowledge_references.length">
<div class="refer_title">
<img src="@/assets/img/ziliao.svg" :alt="$t('chat.referenceIconAlt')" />
<span>{{ $t('chat.referencesTitle', { count: session.knowledge_references?.length ?? 0 }) }}</span>
<span>{{ headerText }}</span>
</div>
<div class="refer_show_icon">
<t-icon :name="showReferBox ? 'chevron-up' : 'chevron-down'" />
</div>
</div>
<div class="refer_box" v-show="showReferBox">
<div v-for="(item, index) in session.knowledge_references" :key="index">
<!-- Web search references: show URL and make it clickable -->
<template v-if="item.chunk_type === 'web_search'">
<a
:href="getWebSearchUrl(item)"
target="_blank"
rel="noopener noreferrer"
class="doc doc-web"
@click.stop
>
{{ session.knowledge_references.length < 2 ? getWebSearchDisplayText(item) : `${index + 1}.${getWebSearchDisplayText(item)}` }}
</a>
</template>
<!-- Regular knowledge references: show title with popup -->
<template v-else>
<t-popup overlayClassName="refer-to-layer" placement="bottom-left" width="400" :showArrow="false"
trigger="click">
<template #content>
<ContentPopup :content="safeProcessContent(item.content)" :is-html="true" />
</template>
<span class="doc">
{{ session.knowledge_references.length < 2 ? item.knowledge_title : `${index +
1}.${item.knowledge_title}` }} </span>
</t-popup>
</template>
<!-- Web search references (ungrouped) -->
<div v-for="(item, index) in webSearchRefs" :key="'web-' + index">
<a
:href="getWebSearchUrl(item)"
target="_blank"
rel="noopener noreferrer"
class="doc doc-web"
@click.stop
>
{{ webSearchRefs.length < 2 ? getWebSearchDisplayText(item) : `${index + 1}. ${getWebSearchDisplayText(item)}` }}
</a>
</div>
<!-- Knowledge references grouped by document -->
<div v-for="(group, gIdx) in groupedKnowledgeRefs" :key="'grp-' + gIdx" class="doc-group">
<div class="doc-group-header" @click="toggleGroup(group.key)">
<div class="doc-group-left">
<t-icon :name="expandedGroups[group.key] ? 'chevron-down' : 'chevron-right'" size="14px" class="doc-group-arrow" />
<t-icon name="file" size="14px" class="doc-group-icon" />
<span class="doc-group-title" :title="group.title">{{ group.title }}</span>
<span class="doc-group-count">{{ $t('chat.referenceChunkCount', { count: group.chunks.length }) }}</span>
</div>
<div class="doc-group-actions" v-if="group.knowledgeBaseId" @click.stop>
<t-tooltip :content="$t('chat.navigateToDocument')">
<span class="doc-group-navigate" @click="navigateToDocument(group)">
<t-icon name="jump" size="14px" />
</span>
</t-tooltip>
</div>
</div>
<div class="doc-group-chunks" v-show="expandedGroups[group.key]">
<div v-for="(chunk, cIdx) in group.chunks" :key="'chunk-' + cIdx" class="doc-chunk-item">
<t-popup overlayClassName="refer-to-layer" placement="bottom-left" width="400" :showArrow="false" trigger="click">
<template #content>
<ContentPopup :content="safeProcessContent(chunk.content)" :is-html="true" />
</template>
<span class="doc-chunk-text">
<span class="doc-chunk-index">{{ $t('chat.chunkLabel', { index: cIdx + 1 }) }}</span>
{{ truncateContent(chunk.content, 80) }}
</span>
</t-popup>
</div>
</div>
</div>
</div>
</div>
</template>
<script setup>
import { onMounted, defineProps, computed, ref, reactive } from "vue";
import { defineProps, computed, ref, reactive } from "vue";
import { useRouter } from 'vue-router';
import { useI18n } from 'vue-i18n';
import { sanitizeHTML } from '@/utils/security';
import ContentPopup from './tool-results/ContentPopup.vue';
const router = useRouter();
const { t } = useI18n();
const props = defineProps({
// 必填项
content: {
type: String,
required: false
@ -54,22 +77,88 @@ const props = defineProps({
required: false
}
});
const showReferBox = ref(false);
const expandedGroups = reactive({});
const referBoxSwitch = () => {
showReferBox.value = !showReferBox.value;
};
// 安全地处理内容
const toggleGroup = (key) => {
expandedGroups[key] = !expandedGroups[key];
};
const webSearchRefs = computed(() => {
if (!props.session?.knowledge_references) return [];
return props.session.knowledge_references.filter(item => item.chunk_type === 'web_search');
});
const knowledgeRefs = computed(() => {
if (!props.session?.knowledge_references) return [];
return props.session.knowledge_references.filter(item => item.chunk_type !== 'web_search');
});
const groupedKnowledgeRefs = computed(() => {
const refs = knowledgeRefs.value;
if (!refs.length) return [];
const groupMap = new Map();
for (const item of refs) {
const key = item.knowledge_id || item.knowledge_title || item.id;
if (!groupMap.has(key)) {
groupMap.set(key, {
key,
title: item.knowledge_title || item.knowledge_filename || key,
knowledgeId: item.knowledge_id,
knowledgeBaseId: item.knowledge_base_id,
chunks: [],
});
}
groupMap.get(key).chunks.push(item);
}
return Array.from(groupMap.values());
});
const headerText = computed(() => {
const total = props.session?.knowledge_references?.length ?? 0;
const docCount = groupedKnowledgeRefs.value.length;
const webCount = webSearchRefs.value.length;
if (docCount > 0 && webCount > 0) {
return t('chat.referencesDocAndWebCount', { docCount, webCount });
}
if (docCount > 0) {
return t('chat.referencesDocCount', { count: docCount });
}
return t('chat.referencesTitle', { count: total });
});
const safeProcessContent = (content) => {
if (!content) return '';
// 先进行安全清理,然后处理换行
const sanitized = sanitizeHTML(content);
return sanitized.replace(/\n/g, '<br/>');
};
// 获取 web_search 类型的 URL
const truncateContent = (content, maxLen) => {
if (!content) return '';
const text = content.replace(/\n/g, ' ').trim();
if (text.length <= maxLen) return text;
return text.slice(0, maxLen) + '...';
};
const navigateToDocument = (group) => {
if (!group.knowledgeBaseId) return;
const query = {};
if (group.knowledgeId) {
query.knowledge_id = group.knowledgeId;
}
router.push({
path: `/platform/knowledge-bases/${group.knowledgeBaseId}`,
query
});
};
const getWebSearchUrl = (item) => {
// 优先使用 metadata.url,其次使用 id(如果 id 是 URL)
if (item.metadata?.url) {
return item.metadata.url;
}
@ -79,16 +168,13 @@ const getWebSearchUrl = (item) => {
return '#';
};
// 获取 web_search 类型的显示文本
const getWebSearchDisplayText = (item) => {
// 优先使用 knowledge_title,其次使用 metadata.title,最后使用 URL 的域名
if (item.knowledge_title) {
return item.knowledge_title;
}
if (item.metadata?.title) {
return item.metadata.title;
}
// 如果都没有,使用 URL 的域名部分
const url = getWebSearchUrl(item);
if (url && url !== '#') {
try {
@ -100,7 +186,6 @@ const getWebSearchDisplayText = (item) => {
}
return 'Web Search Result';
};
</script>
<style lang="less" scoped>
.refer {
@ -155,23 +240,12 @@ const getWebSearchDisplayText = (item) => {
}
.refer_box {
padding: 4px 14px 4px 14px;
padding: 4px 14px 8px 14px;
flex-direction: column;
border-top: 1px solid #f0f0f0;
}
}
.doc_content {
max-height: 400px;
overflow: auto;
font-size: 14px;
color: #000000e6;
line-height: 23px;
text-align: justify;
border: 1px solid #07c05f33;
padding: 8px;
}
.doc {
text-decoration: none;
color: #07c05f;
@ -185,28 +259,133 @@ const getWebSearchDisplayText = (item) => {
padding: 2px 0;
transition: all 0.2s ease;
border-bottom: 1px solid transparent;
&:hover {
border-bottom-color: #07c05f;
}
&.doc-web {
// Web search links can be longer, allow wrapping if needed
white-space: normal;
word-break: break-all;
&:hover {
text-decoration: underline;
}
}
}
.doc-group {
margin-top: 4px;
.doc-group-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 4px 4px;
border-radius: 4px;
cursor: pointer;
transition: background-color 0.15s ease;
&:hover {
background-color: rgba(7, 192, 95, 0.04);
}
.doc-group-left {
display: flex;
align-items: center;
min-width: 0;
flex: 1;
}
.doc-group-arrow {
color: #999;
flex-shrink: 0;
margin-right: 2px;
}
.doc-group-icon {
color: #07c05f;
flex-shrink: 0;
margin-right: 6px;
}
.doc-group-title {
color: #333;
font-weight: 500;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
max-width: 200px;
}
.doc-group-count {
color: #999;
font-size: 11px;
margin-left: 6px;
white-space: nowrap;
flex-shrink: 0;
}
.doc-group-actions {
flex-shrink: 0;
margin-left: 8px;
}
.doc-group-navigate {
display: inline-flex;
align-items: center;
justify-content: center;
width: 22px;
height: 22px;
border-radius: 4px;
color: #07c05f;
cursor: pointer;
transition: all 0.15s ease;
&:hover {
background-color: rgba(7, 192, 95, 0.1);
}
}
}
.doc-group-chunks {
padding-left: 22px;
}
}
.doc-chunk-item {
.doc-chunk-text {
display: block;
color: #666;
font-size: 12px;
line-height: 18px;
padding: 3px 6px;
border-radius: 4px;
cursor: pointer;
transition: background-color 0.15s ease;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
&:hover {
background-color: rgba(7, 192, 95, 0.04);
color: #07c05f;
}
.doc-chunk-index {
color: #999;
font-size: 11px;
margin-right: 4px;
}
}
}
</style>
<style>
.refer-to-layer {
width: 400px;
max-width: 500px;
.t-popup__content {
max-height: 400px;
max-width: 500px;
@ -216,4 +395,4 @@ const getWebSearchDisplayText = (item) => {
word-break: break-word;
}
}
</style>
</style>

View File

@ -31,6 +31,7 @@ import {
import FAQEntryManager from './components/FAQEntryManager.vue';
import { useI18n } from 'vue-i18n';
import { formatStringDate, kbFileTypeVerification } from '@/utils';
import { getParserEngines, type ParserEngineInfo } from '@/api/system';
const route = useRoute();
const { t } = useI18n();
const kbId = computed(() => (route.params as any).kbId as string || '');
@ -40,6 +41,62 @@ const folderUploadInputRef = ref<HTMLInputElement | null>(null);
const uploading = ref(false);
const kbLoading = ref(false);
const isFAQ = computed(() => (kbInfo.value?.type || '') === 'faq');
const parserEngines = ref<ParserEngineInfo[]>([]);
const supportedFileTypes = computed<Set<string>>(() => {
const engines = parserEngines.value
if (!engines.length) return new Set<string>()
const rules: { file_types: string[]; engine: string }[] =
kbInfo.value?.chunking_config?.parser_engine_rules || []
const ruleMap = new Map<string, string>()
for (const r of rules) {
for (const ft of r.file_types) ruleMap.set(ft, r.engine)
}
const available = new Set<string>()
const availableEngineNames = new Set(
engines.filter(e => e.Available !== false).map(e => e.Name)
)
for (const engine of engines) {
for (const ft of engine.FileTypes || []) {
if (available.has(ft)) continue
const explicitEngine = ruleMap.get(ft)
if (explicitEngine) {
if (availableEngineNames.has(explicitEngine)) available.add(ft)
} else {
if (engine.Available !== false) available.add(ft)
}
}
}
return available
})
const acceptFileTypes = computed(() =>
[...supportedFileTypes.value].map(t => '.' + t).join(',')
)
const unsupportedFileTypes = computed<string[]>(() => {
const engines = parserEngines.value
if (!engines.length) return []
const allTypes = new Set<string>()
for (const engine of engines) {
for (const ft of engine.FileTypes || []) allTypes.add(ft)
}
const supported = supportedFileTypes.value
return [...allTypes].filter(ft => !supported.has(ft)).sort()
})
const goToParserSettings = () => {
if (kbId.value) {
uiStore.openKBSettings(kbId.value, 'parser')
}
}
// Permission control: check if current user owns this KB or has edit/manage permission
const isOwner = computed(() => {
@ -121,6 +178,8 @@ const fileTypeOptions = computed(() => [
{ content: 'PDF', value: 'pdf' },
{ content: 'DOCX', value: 'docx' },
{ content: 'DOC', value: 'doc' },
{ content: 'PPTX', value: 'pptx' },
{ content: 'PPT', value: 'ppt' },
{ content: 'TXT', value: 'txt' },
{ content: 'MD', value: 'md' },
{ content: 'URL', value: 'url' },
@ -564,15 +623,35 @@ const handleOpenURLImportDialog = (event: CustomEvent) => {
}
};
// Auto-open document detail when navigated with ?knowledge_id=xxx
const pendingKnowledgeId = ref<string | null>(
(route.query.knowledge_id as string) || null
);
const tryAutoOpenDocument = () => {
if (!pendingKnowledgeId.value || !cardList.value?.length) return;
const targetId = pendingKnowledgeId.value;
pendingKnowledgeId.value = null;
const card = cardList.value.find((c: KnowledgeCard) => c.id === targetId);
if (card) {
nextTick(() => openCardDetails(card));
} else {
nextTick(() => {
openCardDetails({ id: targetId } as KnowledgeCard);
});
}
};
onMounted(() => {
loadKnowledgeBaseInfo(kbId.value);
loadKnowledgeList();
// Load shared knowledge bases to get permission info
orgStore.fetchSharedKnowledgeBases();
// 监听文件上传事件
getParserEngines()
.then(res => { parserEngines.value = res?.data || [] })
.catch(() => { parserEngines.value = [] })
window.addEventListener('knowledgeFileUploaded', handleFileUploaded as EventListener);
// 监听URL导入对话框打开事件
window.addEventListener('openURLImportDialog', handleOpenURLImportDialog as EventListener);
});
@ -582,6 +661,12 @@ onUnmounted(() => {
});
watch(() => cardList.value, (newValue) => {
if (isFAQ.value) return;
// Auto-open document if navigated with ?knowledge_id=xxx
if (pendingKnowledgeId.value && newValue?.length) {
tryAutoOpenDocument();
}
let analyzeList = [];
// Filter items that need polling: parsing in progress OR summary generation in progress
analyzeList = newValue.filter(item => {
@ -780,21 +865,29 @@ const handleDocumentUpload = async (event: Event) => {
return;
}
// 过滤有效文件
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
const validFiles: File[] = [];
let skippedCount = 0;
for (let i = 0; i < files.length; i++) {
const file = files[i];
if (!kbFileTypeVerification(file, files.length > 1)) {
if (!kbFileTypeVerification(file, files.length > 1, dynamicTypes)) {
validFiles.push(file);
} else {
skippedCount++;
}
}
if (validFiles.length === 0) {
if (skippedCount > 0) {
MessagePlugin.warning(`所选文件类型暂无可用解析引擎,已全部跳过`);
}
resetUploadInput();
return;
}
if (skippedCount > 0) {
MessagePlugin.warning(`${skippedCount} 个文件因无可用解析引擎被跳过`);
}
// 批量上传
let successCount = 0;
let failCount = 0;
const totalCount = validFiles.length;
@ -871,10 +964,9 @@ const handleFolderUpload = async (event: Event) => {
return;
}
// 检查是否启用了VLM
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
// 过滤有效文件
const validFiles: File[] = [];
let hiddenFileCount = 0;
let imageFilteredCount = 0;
@ -883,7 +975,6 @@ const handleFolderUpload = async (event: Event) => {
const file = files[i];
const relativePath = (file as any).webkitRelativePath || file.name;
// 1. 过滤隐藏文件和隐藏文件夹
const pathParts = relativePath.split('/');
const hasHiddenComponent = pathParts.some((part: string) => part.startsWith('.'));
if (hasHiddenComponent) {
@ -891,7 +982,6 @@ const handleFolderUpload = async (event: Event) => {
continue;
}
// 2. 如果未启用VLM,过滤图片文件
if (!vlmEnabled) {
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
@ -901,8 +991,7 @@ const handleFolderUpload = async (event: Event) => {
}
}
// 3. 文件类型验证
if (!kbFileTypeVerification(file, true)) {
if (!kbFileTypeVerification(file, true, dynamicTypes)) {
validFiles.push(file);
}
}
@ -1111,9 +1200,8 @@ const delCardConfirm = () => {
// 处理知识库编辑成功后的回调
const handleKBEditorSuccess = (kbIdValue: string) => {
// 如果编辑的是当前知识库,刷新文件列表
if (kbIdValue === kbId.value) {
loadKnowledgeFiles(kbIdValue);
loadKnowledgeBaseInfo(kbIdValue);
}
};
@ -1226,6 +1314,11 @@ async function createNewSession(value: string): Promise<void> {
</t-tooltip>
</div>
<p class="document-subtitle">{{ $t('knowledgeEditor.document.subtitle') }}</p>
<p v-if="unsupportedFileTypes.length" class="parser-hint" @click="goToParserSettings">
<t-icon name="info-circle" class="parser-hint-icon" />
<span>{{ $t('knowledgeBase.unsupportedTypesHint', { types: unsupportedFileTypes.map(t => '.' + t).join('、') }) }}</span>
<span class="parser-hint-link">{{ $t('knowledgeBase.goToParserSettings') }} →</span>
</p>
</div>
</div>
@ -1233,7 +1326,7 @@ async function createNewSession(value: string): Promise<void> {
ref="uploadInputRef"
type="file"
class="document-upload-input"
accept=".pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xlsx,.xls"
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'"
multiple
@change="handleDocumentUpload"
/>
@ -2197,7 +2290,7 @@ async function createNewSession(value: string): Promise<void> {
background: transparent;
border: none;
&:hover {
color: #4e5969;
color: #07C05F;
background: #f2f3f5;
}
}
@ -2212,7 +2305,7 @@ async function createNewSession(value: string): Promise<void> {
&:hover,
&:focus,
&.t-is-focused {
border-color: #4080ff;
border-color: #07C05F;
background-color: #fff;
}
}
@ -2226,7 +2319,7 @@ async function createNewSession(value: string): Promise<void> {
&:hover,
&.t-is-focused {
border-color: #4080ff;
border-color: #07C05F;
background-color: #fff;
}
}
@ -2375,6 +2468,36 @@ async function createNewSession(value: string): Promise<void> {
line-height: 20px;
}
.parser-hint {
display: flex;
align-items: center;
gap: 4px;
margin: 2px 0 0;
color: #b08000;
font-size: 12px;
line-height: 1.4;
cursor: pointer;
transition: color 0.15s ease;
&:hover {
color: #8a6300;
.parser-hint-link {
text-decoration: underline;
}
}
.parser-hint-icon {
font-size: 12px;
flex-shrink: 0;
}
.parser-hint-link {
color: #07C05F;
margin-left: 2px;
white-space: nowrap;
}
}
}

View File

@ -119,6 +119,22 @@
</div>
</div>
<!-- 解析引擎 -->
<div v-if="!isFAQ && formData" v-show="currentSection === 'parser'" class="section">
<KBParserSettings
:parser-engine-rules="formData.chunkingConfig.parserEngineRules"
@update:parser-engine-rules="handleParserEngineRulesUpdate"
/>
</div>
<!-- 存储引擎 -->
<div v-if="!isFAQ && formData" v-show="currentSection === 'storage'" class="section">
<KBStorageSettings
:storage-provider="formData.storageProvider"
@update:storage-provider="handleStorageProviderUpdate"
/>
</div>
<!-- 分块设置 -->
<div v-if="!isFAQ" v-show="currentSection === 'chunking'" class="section">
<KBChunkingSettings
@ -182,6 +198,8 @@ import { updateKBConfig, type KBModelConfigRequest } from '@/api/initialization'
import { listModels } from '@/api/model'
import { useUIStore } from '@/stores/ui'
import KBModelConfig from './settings/KBModelConfig.vue'
import KBParserSettings from './settings/KBParserSettings.vue'
import KBStorageSettings from './settings/KBStorageSettings.vue'
import KBChunkingSettings from './settings/KBChunkingSettings.vue'
import KBAdvancedSettings from './settings/KBAdvancedSettings.vue'
import GraphSettings from './settings/GraphSettings.vue'
@ -220,6 +238,8 @@ const navItems = computed(() => {
items.push({ key: 'faq', icon: 'help-circle', label: t('knowledgeEditor.sidebar.faq') })
} else {
items.push(
{ key: 'parser', icon: 'file-search', label: '解析引擎' },
{ key: 'storage', icon: 'cloud', label: t('knowledgeEditor.sidebar.storage') },
{ key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') },
{ key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') },
{ key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') }
@ -274,25 +294,13 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
chunkingConfig: {
chunkSize: 512,
chunkOverlap: 100,
separators: ['\n\n', '\n', '。', '!', '?', ';', ';']
separators: ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: undefined as any
},
storageProvider: 'local' as string,
multimodalConfig: {
enabled: false,
storageType: 'minio' as 'minio' | 'cos',
vllmModelId: '',
minio: {
bucketName: '',
useSSL: false,
pathPrefix: ''
},
cos: {
secretId: '',
secretKey: '',
region: '',
bucketName: '',
appId: '',
pathPrefix: ''
}
vllmModelId: ''
},
nodeExtractConfig: {
enabled: false,
@ -363,25 +371,13 @@ const loadKBData = async () => {
chunkingConfig: {
chunkSize: kb.chunking_config?.chunk_size || 512,
chunkOverlap: kb.chunking_config?.chunk_overlap || 100,
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';']
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined
},
storageProvider: (kb.cos_config?.provider || 'local') as string,
multimodalConfig: {
enabled: !!(kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)),
storageType: (kb.cos_config?.provider || 'minio') as 'minio' | 'cos',
vllmModelId: kb.vlm_config?.model_id || '',
minio: {
bucketName: kb.cos_config?.bucket_name || '',
useSSL: kb.cos_config?.use_ssl || false,
pathPrefix: kb.cos_config?.path_prefix || ''
},
cos: {
secretId: kb.cos_config?.secret_id || '',
secretKey: kb.cos_config?.secret_key || '',
region: kb.cos_config?.region || '',
bucketName: kb.cos_config?.bucket_name || '',
appId: kb.cos_config?.app_id || '',
pathPrefix: kb.cos_config?.path_prefix || ''
}
enabled: !!kb.vlm_config?.enabled,
vllmModelId: kb.vlm_config?.model_id || ''
},
nodeExtractConfig: {
enabled: kb.extract_config?.enabled || false,
@ -420,12 +416,24 @@ const handleChunkingConfigUpdate = (config: any) => {
}
}
const handleParserEngineRulesUpdate = (rules: any[]) => {
if (formData.value) {
formData.value.chunkingConfig.parserEngineRules = rules?.length ? rules : undefined
}
}
const handleMultimodalUpdate = (config: any) => {
if (formData.value) {
formData.value.multimodalConfig = { ...config }
}
}
const handleStorageProviderUpdate = (value: string) => {
if (formData.value) {
formData.value.storageProvider = value || 'local'
}
}
const handleQuestionGenerationUpdate = (config: any) => {
if (formData.value) {
formData.value.questionGenerationConfig = { ...config }
@ -493,7 +501,10 @@ const buildSubmitData = () => {
chunk_size: formData.value.chunkingConfig.chunkSize,
chunk_overlap: formData.value.chunkingConfig.chunkOverlap,
separators: formData.value.chunkingConfig.separators,
enable_multimodal: formData.value.multimodalConfig.enabled
enable_multimodal: formData.value.multimodalConfig.enabled,
...(formData.value.chunkingConfig.parserEngineRules?.length
? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules }
: {})
},
embedding_model_id: formData.value.modelConfig.embeddingModelId,
summary_model_id: formData.value.modelConfig.llmModelId
@ -507,26 +518,9 @@ const buildSubmitData = () => {
: ''
}
if (formData.value.multimodalConfig.enabled) {
const storageType = formData.value.multimodalConfig.storageType
if (storageType === 'minio') {
data.cos_config = {
provider: 'minio',
bucket_name: formData.value.multimodalConfig.minio.bucketName,
use_ssl: formData.value.multimodalConfig.minio.useSSL,
path_prefix: formData.value.multimodalConfig.minio.pathPrefix || undefined
}
} else {
data.cos_config = {
provider: 'cos',
secret_id: formData.value.multimodalConfig.cos.secretId,
secret_key: formData.value.multimodalConfig.cos.secretKey,
region: formData.value.multimodalConfig.cos.region,
bucket_name: formData.value.multimodalConfig.cos.bucketName,
app_id: formData.value.multimodalConfig.cos.appId,
path_prefix: formData.value.multimodalConfig.cos.pathPrefix || undefined
}
}
// 存储引擎:仅传 provider,参数从全局设置读取
data.cos_config = {
provider: formData.value.storageProvider || 'local'
}
// 添加知识图谱配置
@ -599,7 +593,7 @@ const handleSubmit = async () => {
config: updateConfig
})
// 2. 更新完整配置(模型、分块、多模态、知识图谱等)
// 2. 更新完整配置(模型、分块、多模态、存储引擎、知识图谱等)
const config: KBModelConfigRequest = {
llmModelId: data.summary_model_id,
embeddingModelId: data.embedding_model_id,
@ -607,25 +601,13 @@ const handleSubmit = async () => {
documentSplitting: {
chunkSize: data.chunking_config.chunk_size,
chunkOverlap: data.chunking_config.chunk_overlap,
separators: data.chunking_config.separators
separators: data.chunking_config.separators,
parserEngineRules: data.chunking_config.parser_engine_rules || undefined
},
multimodal: {
enabled: !!data.cos_config || !!data.vlm_config?.enabled,
storageType: data.cos_config?.provider || 'minio',
cos: data.cos_config?.provider === 'cos' ? {
secretId: data.cos_config.secret_id,
secretKey: data.cos_config.secret_key,
region: data.cos_config.region,
bucketName: data.cos_config.bucket_name,
appId: data.cos_config.app_id,
pathPrefix: data.cos_config.path_prefix || ''
} : undefined,
minio: data.cos_config?.provider === 'minio' ? {
bucketName: data.cos_config.bucket_name,
useSSL: data.cos_config.use_ssl || false,
pathPrefix: data.cos_config.path_prefix || ''
} : undefined
enabled: !!data.vlm_config?.enabled
},
storageProvider: data.cos_config?.provider || 'local',
nodeExtract: {
enabled: data.extract_config?.enabled || false,
text: data.extract_config?.text || '',

View File

@ -42,7 +42,7 @@
</div>
</div>
<!-- Multimodal feature -->
<!-- Multimodal feature:仅选择多模态模型,存储引擎在「存储引擎」页配置 -->
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.label') }}</label>
@ -57,9 +57,8 @@
</div>
</div>
<!-- Multimodal storage configuration -->
<!-- 多模态开启时仅配置 VLLM 模型 -->
<div v-if="localMultimodal.enabled" class="subsection">
<!-- VLLM model -->
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.vllmLabel') }} <span class="required">*</span></label>
@ -77,206 +76,6 @@
/>
</div>
</div>
<div class="subsection-header">
<h4>{{ $t('knowledgeEditor.advanced.multimodal.storageTitle') }} <span class="required">*</span></h4>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.storageTypeLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.storageTypeDescription') }}</p>
<!-- Warning message when MinIO is not enabled -->
<t-alert
v-if="!isMinioEnabled"
theme="warning"
:message="$t('knowledgeEditor.advanced.multimodal.minioDisabledWarning')"
style="margin-top: 8px;"
/>
</div>
<div class="setting-control">
<t-radio-group v-model="localMultimodal.storageType" @change="handleStorageTypeChange">
<t-radio value="minio" :disabled="!isMinioEnabled">
{{ $t('knowledgeEditor.advanced.multimodal.storageTypeOptions.minio') }}
</t-radio>
<t-radio value="cos">{{ $t('knowledgeEditor.advanced.multimodal.storageTypeOptions.cos') }}</t-radio>
</t-radio-group>
</div>
</div>
<!-- MinIO configuration -->
<div v-if="localMultimodal.storageType === 'minio'" class="storage-config">
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.minio.bucketLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.minio.bucketDescription') }}</p>
<p class="hint">{{ $t('knowledgeEditor.advanced.multimodal.minio.bucketHint') }}</p>
</div>
<div class="setting-control bucket-control">
<t-select
v-model="localMultimodal.minio.bucketName"
:placeholder="$t('knowledgeEditor.advanced.multimodal.minio.bucketPlaceholder')"
:loading="loadingBuckets"
filterable
creatable
@change="handleConfigChange"
@focus="loadMinioBuckets"
style="width: 280px;"
>
<t-option
v-for="bucket in minioBuckets"
:key="bucket.name"
:value="bucket.name"
:label="bucket.name"
>
<div class="bucket-option">
<span class="bucket-name">{{ bucket.name }}</span>
<t-tag
:theme="bucket.policy === 'public' ? 'success' : bucket.policy === 'private' ? 'default' : 'warning'"
size="small"
variant="light"
>
{{ $t(`knowledgeEditor.advanced.multimodal.minio.policyLabels.${bucket.policy}`) }}
</t-tag>
</div>
</t-option>
</t-select>
<t-button
theme="default"
variant="outline"
size="small"
:loading="loadingBuckets"
@click="loadMinioBuckets"
style="margin-left: 8px;"
>
<template #icon><refresh-icon /></template>
</t-button>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.minio.useSslLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.minio.useSslDescription') }}</p>
</div>
<div class="setting-control">
<t-switch
v-model="localMultimodal.minio.useSSL"
@change="handleConfigChange"
size="large"
/>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.minio.pathPrefixLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.minio.pathPrefixDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.minio.pathPrefix"
:placeholder="$t('knowledgeEditor.advanced.multimodal.minio.pathPrefixPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
</div>
<!-- COS configuration -->
<div v-if="localMultimodal.storageType === 'cos'" class="storage-config">
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.secretIdLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.secretIdDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.cos.secretId"
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.secretIdPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.secretKeyLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.secretKeyDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.cos.secretKey"
type="password"
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.secretKeyPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.regionLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.regionDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.cos.region"
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.regionPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.bucketLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.bucketDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.cos.bucketName"
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.bucketPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.appIdLabel') }} <span class="required">*</span></label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.appIdDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.cos.appId"
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.appIdPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.pathPrefixLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.pathPrefixDescription') }}</p>
</div>
<div class="setting-control">
<t-input
v-model="localMultimodal.cos.pathPrefix"
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.pathPrefixPlaceholder')"
@change="handleConfigChange"
style="width: 280px;"
/>
</div>
</div>
</div>
</div>
</div>
@ -284,31 +83,15 @@
</template>
<script setup lang="ts">
import { ref, watch, onMounted } from 'vue'
import { RefreshIcon } from 'tdesign-icons-vue-next'
import { ref, watch } from 'vue'
import ModelSelector from '@/components/ModelSelector.vue'
import { useUIStore } from '@/stores/ui'
import { getSystemInfo, listMinioBuckets, type MinioBucketInfo } from '@/api/system'
const uiStore = useUIStore()
interface MultimodalConfig {
enabled: boolean
storageType: 'minio' | 'cos'
vllmModelId?: string
minio: {
bucketName: string
useSSL: boolean
pathPrefix: string
}
cos: {
secretId: string
secretKey: string
region: string
bucketName: string
appId: string
pathPrefix: string
}
}
interface QuestionGenerationConfig {
@ -335,58 +118,7 @@ const localQuestionGeneration = ref<QuestionGenerationConfig>(
)
const vllmSelectorRef = ref()
const isMinioEnabled = ref(false)
const minioBuckets = ref<MinioBucketInfo[]>([])
const loadingBuckets = ref(false)
// Load MinIO buckets
const loadMinioBuckets = async () => {
if (!isMinioEnabled.value || loadingBuckets.value) return
loadingBuckets.value = true
try {
const response = await listMinioBuckets()
if (response.data?.buckets) {
minioBuckets.value = response.data.buckets
}
} catch (error) {
console.error('Failed to load MinIO buckets:', error)
} finally {
loadingBuckets.value = false
}
}
// Check system status on mount
onMounted(async () => {
try {
const systemInfo = await getSystemInfo()
// Check MinIO status
isMinioEnabled.value = systemInfo.data?.minio_enabled === true
// If MinIO is not enabled and storage type is minio, switch to cos
if (!isMinioEnabled.value && localMultimodal.value.storageType === 'minio') {
localMultimodal.value.storageType = 'cos'
emit('update:multimodal', localMultimodal.value)
}
// Load MinIO buckets if enabled
if (isMinioEnabled.value) {
loadMinioBuckets()
}
} catch (error) {
console.error('Failed to fetch system info:', error)
// Default to disabled if we can't fetch the info
isMinioEnabled.value = false
// If MinIO status unknown and storage type is minio, switch to cos
if (localMultimodal.value.storageType === 'minio') {
localMultimodal.value.storageType = 'cos'
emit('update:multimodal', localMultimodal.value)
}
}
})
// Watch for prop changes
watch(() => props.multimodal, (newVal) => {
localMultimodal.value = { ...newVal }
}, { deep: true })
@ -397,7 +129,6 @@ watch(() => props.questionGeneration, (newVal) => {
}
}, { deep: true })
// Handle question generation toggle
const handleQuestionGenerationToggle = () => {
if (!localQuestionGeneration.value.enabled) {
localQuestionGeneration.value.questionCount = 3
@ -405,59 +136,25 @@ const handleQuestionGenerationToggle = () => {
emit('update:questionGeneration', localQuestionGeneration.value)
}
// Handle question generation config change
const handleQuestionGenerationChange = () => {
emit('update:questionGeneration', localQuestionGeneration.value)
}
// Handle multimodal toggle
const handleMultimodalToggle = () => {
// Reset related configuration when multimodal is disabled
if (!localMultimodal.value.enabled) {
localMultimodal.value.vllmModelId = ''
localMultimodal.value.minio = {
bucketName: '',
useSSL: false,
pathPrefix: ''
}
localMultimodal.value.cos = {
secretId: '',
secretKey: '',
region: '',
bucketName: '',
appId: '',
pathPrefix: ''
}
}
emit('update:multimodal', localMultimodal.value)
}
// Handle storage type change
const handleStorageTypeChange = () => {
// Prevent switching to minio if it's not enabled
if (localMultimodal.value.storageType === 'minio' && !isMinioEnabled.value) {
localMultimodal.value.storageType = 'cos'
}
emit('update:multimodal', localMultimodal.value)
}
// Handle VLLM model change
const handleVLLMChange = (modelId: string) => {
localMultimodal.value.vllmModelId = modelId
emit('update:multimodal', localMultimodal.value)
}
// Navigate to model management when adding models
const handleAddModel = (subSection: string) => {
uiStore.openSettings('models', subSection)
}
// Handle configuration change
const handleConfigChange = () => {
emit('update:multimodal', localMultimodal.value)
}
// The allModels prop keeps model options in sync
</script>
<style lang="less" scoped>
@ -546,104 +243,11 @@ const handleConfigChange = () => {
position: relative;
}
.subsection-header {
margin: 16px 0 8px 0;
&:first-child {
margin-top: 0;
}
h4 {
font-size: 15px;
font-weight: 600;
color: #333333;
margin: 0;
padding-left: 8px;
border-left: 2px solid #07C05F;
.required {
color: #e34d59;
margin-left: 4px;
}
}
}
.required {
color: #e34d59;
margin-left: 2px;
font-weight: 500;
}
.storage-config {
margin-top: 8px;
}
.bucket-control {
display: flex;
align-items: center;
}
.bucket-option {
display: flex;
align-items: center;
justify-content: space-between;
width: 100%;
.bucket-name {
flex: 1;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}
.config-item {
background: #ffffff;
border: 1px solid #e5e7eb;
border-radius: 6px;
padding: 16px;
margin-bottom: 12px;
}
.config-item-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 16px;
padding-bottom: 12px;
border-bottom: 1px solid #f0f0f0;
}
.config-item-title {
font-size: 14px;
font-weight: 600;
color: #333333;
}
.config-item-body {
display: flex;
flex-direction: column;
gap: 12px;
}
.config-field {
display: flex;
flex-direction: column;
gap: 6px;
label {
font-size: 13px;
font-weight: 500;
color: #555555;
}
}
.subsection-desc {
font-size: 13px;
color: #666666;
margin: 4px 0 8px 8px;
line-height: 1.5;
}
</style>

View File

@ -77,10 +77,16 @@
import { ref, watch, computed } from 'vue'
import { useI18n } from 'vue-i18n'
interface ParserEngineRule {
file_types: string[]
engine: string
}
interface ChunkingConfig {
chunkSize: number
chunkOverlap: number
separators: string[]
parserEngineRules?: ParserEngineRule[]
}
interface Props {
@ -98,7 +104,6 @@ const localChunkOverlap = ref(props.config.chunkOverlap)
const localSeparators = ref([...props.config.separators])
const { t } = useI18n()
// Separator options
const separatorOptions = computed(() => [
{ label: t('knowledgeEditor.chunking.separators.doubleNewline'), value: '\n\n' },
{ label: t('knowledgeEditor.chunking.separators.singleNewline'), value: '\n' },
@ -110,34 +115,22 @@ const separatorOptions = computed(() => [
{ label: t('knowledgeEditor.chunking.separators.space'), value: ' ' }
])
// Watch for prop changes
watch(() => props.config, (newConfig) => {
localChunkSize.value = newConfig.chunkSize
localChunkOverlap.value = newConfig.chunkOverlap
localSeparators.value = [...newConfig.separators]
}, { deep: true })
// Handle chunk size change
const handleChunkSizeChange = () => {
emitUpdate()
}
const handleChunkSizeChange = () => { emitUpdate() }
const handleChunkOverlapChange = () => { emitUpdate() }
const handleSeparatorsChange = () => { emitUpdate() }
// Handle chunk overlap change
const handleChunkOverlapChange = () => {
emitUpdate()
}
// Handle separator change
const handleSeparatorsChange = () => {
emitUpdate()
}
// Emit update event
const emitUpdate = () => {
emit('update:config', {
chunkSize: localChunkSize.value,
chunkOverlap: localChunkOverlap.value,
separators: localSeparators.value
separators: localSeparators.value,
parserEngineRules: props.config.parserEngineRules
})
}
</script>
@ -228,4 +221,3 @@ const emitUpdate = () => {
text-align: right;
}
</style>

View File

@ -0,0 +1,456 @@
<template>
<div class="kb-parser-settings">
<div class="section-header">
<h2>解析引擎</h2>
<p class="section-description">为不同文件类型选择文档解析引擎。未配置的文件类型将使用内置解析引擎。</p>
</div>
<div v-if="loading" class="loading-inline">
<t-loading size="small" />
<span>加载中...</span>
</div>
<div v-else-if="fileTypeGroups.length === 0" class="empty-hint">
<p>暂无可用解析引擎,或文档解析服务未配置。</p>
</div>
<div v-else class="settings-group">
<div
v-for="group in fileTypeGroups"
:key="group.key"
class="setting-row"
>
<div class="setting-info">
<label class="group-label">
<t-icon :name="group.icon" class="group-icon" />
{{ group.label }}
</label>
<div class="ext-tags">
<span v-for="ext in group.extensions" :key="ext" class="ext-tag">.{{ ext }}</span>
</div>
</div>
<div class="setting-control">
<t-select
:value="getEngineForGroup(group.extensions) || undefined"
@change="(val: string) => handleEngineChange(group.extensions, val)"
style="width: 280px;"
:status="hasAvailableEngine(group.extensions) ? 'default' : 'warning'"
placeholder="无可用引擎"
>
<t-option
v-for="opt in getEngineOptions(group.extensions)"
:key="opt.value"
:value="opt.value"
:label="opt.selectLabel"
:disabled="opt.disabled"
>
<t-tooltip
:content="`支持格式: ${opt.fileTypes.map(t => '.' + t).join(' ')}`"
placement="left"
:show-arrow="false"
>
<div class="engine-option">
<div class="engine-option-top">
<span class="engine-option-name">{{ opt.value }}</span>
<t-tag
v-if="opt.isDefault"
theme="primary"
variant="light"
size="small"
>默认</t-tag>
<t-tag
v-if="opt.disabled"
theme="danger"
variant="light"
size="small"
>不可用</t-tag>
</div>
<div class="engine-option-desc">{{ opt.desc }}</div>
<div v-if="opt.disabled && opt.reason" class="engine-option-reason">
{{ opt.reason }}
<a class="go-settings" @click.stop.prevent="goToParserSettings">去设置 →</a>
</div>
</div>
</t-tooltip>
</t-option>
</t-select>
<div v-if="!hasAvailableEngine(group.extensions)" class="no-engine-warning">
<a class="go-settings" @click.prevent="goToParserSettings">前往配置 →</a>
</div>
</div>
</div>
</div>
</div>
</template>
<script setup lang="ts">
import { ref, watch, computed, onMounted, onUnmounted } from 'vue'
import { getParserEngines, type ParserEngineInfo } from '@/api/system'
import { useUIStore } from '@/stores/ui'
import { storeToRefs } from 'pinia'
export interface ParserEngineRule {
file_types: string[]
engine: string
}
interface EngineOption {
value: string
selectLabel: string
desc: string
fileTypes: string[]
disabled: boolean
isDefault: boolean
reason?: string
}
interface Props {
parserEngineRules?: ParserEngineRule[]
}
const props = withDefaults(defineProps<Props>(), {
parserEngineRules: () => []
})
const emit = defineEmits<{
'update:parserEngineRules': [value: ParserEngineRule[]]
}>()
const uiStore = useUIStore()
const localEngineRules = ref<ParserEngineRule[]>([...props.parserEngineRules])
const parserEngines = ref<ParserEngineInfo[]>([])
const loading = ref(true)
const allFileTypes = computed(() => {
const s = new Set<string>()
for (const engine of parserEngines.value) {
for (const ft of engine.FileTypes || []) {
s.add(ft)
}
}
return s
})
const fileTypeGroups = computed(() => {
const ft = allFileTypes.value
const groups: { key: string; label: string; icon: string; extensions: string[] }[] = []
const pdfExts = ['pdf'].filter(e => ft.has(e))
const officeExts = ['docx', 'doc'].filter(e => ft.has(e))
const pptExts = ['pptx', 'ppt'].filter(e => ft.has(e))
const excelExts = ['xlsx', 'xls'].filter(e => ft.has(e))
const csvExts = ['csv'].filter(e => ft.has(e))
const mdExts = ['md', 'markdown'].filter(e => ft.has(e))
const txtExts = ['txt'].filter(e => ft.has(e))
const imageExts = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp'].filter(e => ft.has(e))
if (pdfExts.length) groups.push({ key: 'pdf', label: 'PDF 文档', icon: 'file-pdf', extensions: pdfExts })
if (officeExts.length) groups.push({ key: 'office', label: 'Word 文档', icon: 'file-word', extensions: officeExts })
if (pptExts.length) groups.push({ key: 'ppt', label: '演示文稿', icon: 'file-powerpoint', extensions: pptExts })
if (excelExts.length) groups.push({ key: 'excel', label: 'Excel 表格', icon: 'file-excel', extensions: excelExts })
if (csvExts.length) groups.push({ key: 'csv', label: 'CSV 文件', icon: 'file-excel', extensions: csvExts })
if (mdExts.length) groups.push({ key: 'markdown', label: 'Markdown', icon: 'file-code', extensions: mdExts })
if (txtExts.length) groups.push({ key: 'text', label: '纯文本', icon: 'file', extensions: txtExts })
if (imageExts.length) groups.push({ key: 'image', label: '图片', icon: 'image', extensions: imageExts })
return groups
})
function getEngineOptions(extensions: string[]): EngineOption[] {
const raw: { name: string; desc: string; fileTypes: string[]; available: boolean; reason: string }[] = []
for (const engine of parserEngines.value) {
const supports = extensions.some(ext => (engine.FileTypes || []).includes(ext))
if (supports) {
raw.push({
name: engine.Name,
desc: engine.Description || engine.Name,
fileTypes: engine.FileTypes || [],
available: engine.Available !== false,
reason: engine.UnavailableReason || '',
})
}
}
const defaultName = raw.find(e => e.available)?.name ?? ''
return raw.map(e => ({
value: e.name,
selectLabel: `${e.name} — ${e.desc}`,
desc: e.desc,
fileTypes: e.fileTypes,
disabled: !e.available,
isDefault: defaultName !== '' && e.name === defaultName,
reason: e.reason,
}))
}
function hasAvailableEngine(extensions: string[]): boolean {
return getEngineOptions(extensions).some(opt => !opt.disabled)
}
function getDefaultEngine(extensions: string[]): string {
const opts = getEngineOptions(extensions)
return opts.find(o => o.isDefault)?.value ?? ''
}
function getEngineForGroup(extensions: string[]): string {
for (const rule of localEngineRules.value) {
if (rule.file_types.some(ft => extensions.includes(ft))) {
return rule.engine
}
}
return getDefaultEngine(extensions)
}
function handleEngineChange(extensions: string[], engine: string) {
const otherRules = localEngineRules.value.filter(
r => !r.file_types.some(ft => extensions.includes(ft))
)
if (engine) {
otherRules.push({ file_types: [...extensions], engine })
}
localEngineRules.value = otherRules
emit('update:parserEngineRules', buildCompleteRules())
}
function buildCompleteRules(): ParserEngineRule[] {
const rules: ParserEngineRule[] = []
for (const group of fileTypeGroups.value) {
const engine = getEngineForGroup(group.extensions)
if (engine) {
rules.push({ file_types: [...group.extensions], engine })
}
}
return rules
}
function goToParserSettings() {
uiStore.openSettings('parser')
}
async function loadEngines() {
loading.value = true
try {
const resp = await getParserEngines()
if (resp?.data && Array.isArray(resp.data)) {
parserEngines.value = resp.data
}
} catch {
parserEngines.value = []
} finally {
loading.value = false
ensureCompleteRules()
}
}
function ensureCompleteRules() {
if (!parserEngines.value.length) return
const complete = buildCompleteRules()
if (complete.length && complete.length > localEngineRules.value.length) {
localEngineRules.value = complete
emit('update:parserEngineRules', complete)
}
}
onMounted(loadEngines)
const { showSettingsModal } = storeToRefs(uiStore)
watch(showSettingsModal, (open, wasOpen) => {
if (wasOpen && !open) {
loadEngines()
}
})
watch(() => props.parserEngineRules, (v) => {
localEngineRules.value = v?.length ? [...v] : []
}, { deep: true })
</script>
<style lang="less" scoped>
.kb-parser-settings {
width: 100%;
}
.section-header {
margin-bottom: 32px;
h2 {
font-size: 20px;
font-weight: 600;
color: #333;
margin: 0 0 8px 0;
}
.section-description {
font-size: 14px;
color: #666;
margin: 0;
line-height: 1.5;
}
}
.loading-inline {
display: flex;
align-items: center;
gap: 8px;
padding: 16px 0;
}
.empty-hint {
padding: 24px 0;
color: #666;
}
.settings-group {
display: flex;
flex-direction: column;
gap: 0;
}
.setting-row {
display: flex;
align-items: flex-start;
justify-content: space-between;
padding: 20px 0;
border-bottom: 1px solid #e5e7eb;
&:last-child {
border-bottom: none;
}
}
.setting-info {
flex: 1;
max-width: 65%;
padding-right: 24px;
.group-label {
display: flex;
align-items: center;
gap: 6px;
}
.group-icon {
font-size: 18px;
color: #555;
flex-shrink: 0;
}
label {
font-size: 15px;
font-weight: 500;
color: #333;
display: block;
margin-bottom: 4px;
}
.ext-tags {
display: flex;
flex-wrap: wrap;
gap: 6px;
margin-top: 6px;
}
.ext-tag {
display: inline-block;
font-size: 12px;
line-height: 1;
color: #555;
background: #f3f4f6;
padding: 3px 8px;
border-radius: 4px;
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
}
.desc {
font-size: 13px;
color: #666;
margin: 0;
line-height: 1.5;
}
}
.setting-control {
flex-shrink: 0;
min-width: 280px;
display: flex;
flex-direction: column;
align-items: flex-end;
}
.no-engine-warning {
display: flex;
align-items: center;
gap: 4px;
margin-top: 8px;
font-size: 12px;
color: #e37318;
line-height: 1.4;
.go-settings {
color: #07C05F;
cursor: pointer;
white-space: nowrap;
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
}
// ---- 下拉选项样式 ----
.engine-option {
display: flex;
flex-direction: column;
gap: 3px;
padding: 3px 0;
}
.engine-option-top {
display: flex;
align-items: center;
gap: 6px;
}
.engine-option-name {
font-size: 13px;
font-weight: 600;
color: #333;
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
}
.engine-option-desc {
font-size: 12px;
color: #888;
line-height: 1.4;
}
.engine-option-reason {
font-size: 12px;
color: #e34d59;
line-height: 1.4;
.go-settings {
color: #07C05F;
cursor: pointer;
margin-left: 4px;
font-size: 12px;
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
}
</style>
<style lang="less">
.t-select__dropdown .t-select-option {
height: auto;
align-items: flex-start;
padding-top: 6px;
padding-bottom: 6px;
}
.t-select__dropdown .t-select-option__content {
white-space: normal;
}
</style>

View File

@ -0,0 +1,244 @@
<template>
<div class="kb-storage-settings">
<div class="section-header">
<h2>存储引擎</h2>
<p class="section-description">
选择文件存储引擎,影响文档上传存储和文档中图片的存储方式。参数在全局设置中配置。
</p>
</div>
<div v-if="loading" class="loading-inline">
<t-loading size="small" />
<span>加载中...</span>
</div>
<div v-else class="settings-group">
<div class="setting-row">
<div class="setting-info">
<label>存储引擎</label>
<p class="desc">选择该知识库使用的存储引擎,需在全局设置中已配置对应引擎。</p>
</div>
<div class="setting-control">
<t-select
v-model="localProvider"
size="medium"
placeholder="请选择存储引擎"
style="width: 100%; min-width: 220px;"
@change="handleChange"
>
<t-option
v-for="opt in engineOptions"
:key="opt.value"
:value="opt.value"
:label="opt.label"
:disabled="opt.disabled"
>
<span class="select-option">
<span>{{ opt.label }}</span>
<t-tag v-if="opt.disabled" theme="warning" variant="light" size="small">未配置</t-tag>
<t-tag v-else-if="opt.available === false" theme="danger" variant="light" size="small">不可用</t-tag>
</span>
</t-option>
</t-select>
<p v-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
<a v-if="showGoSettings" href="javascript:void(0)" class="go-settings" @click.prevent="goToStorageSettings">去全局设置中配置</a>
</div>
</div>
</div>
</div>
</template>
<script setup lang="ts">
import { ref, computed, watch, onMounted } from 'vue'
import { getStorageEngineConfig, getStorageEngineStatus, type StorageEngineStatusItem } from '@/api/system'
import { useUIStore } from '@/stores/ui'
const props = defineProps<{
storageProvider: string
}>()
const emit = defineEmits<{
'update:storageProvider': [value: string]
}>()
const uiStore = useUIStore()
const localProvider = ref(props.storageProvider || 'local')
const loading = ref(true)
const engineStatus = ref<StorageEngineStatusItem[]>([])
const defaultProvider = ref('local')
const hasAnyConfig = ref(false)
const engineOptions = computed(() => {
const statusMap: Record<string, boolean> = {}
for (const e of engineStatus.value) {
statusMap[e.name] = e.available
}
return [
{
value: 'local',
label: 'Local(本地存储)',
desc: '仅适合单机部署,简单轻量',
available: statusMap.local !== false,
disabled: false,
},
{
value: 'minio',
label: 'MinIO',
desc: 'S3 兼容,适合内网或私有云',
available: statusMap.minio,
disabled: statusMap.minio === false,
},
{
value: 'cos',
label: '腾讯云 COS',
desc: '公有云部署,支持 CDN 加速',
available: statusMap.cos,
disabled: statusMap.cos === false,
},
]
})
const showGoSettings = computed(() =>
engineOptions.value.some(o => o.disabled)
)
const selectedOption = computed(() =>
engineOptions.value.find(o => o.value === localProvider.value)
)
function handleChange() {
emit('update:storageProvider', localProvider.value)
}
function goToStorageSettings() {
uiStore.closeKBEditor?.()
uiStore.openSettings?.('storage')
}
async function load() {
loading.value = true
try {
const [configRes, statusRes] = await Promise.all([
getStorageEngineConfig(),
getStorageEngineStatus(),
])
const engines = statusRes?.data?.engines ?? []
engineStatus.value = engines
defaultProvider.value = configRes?.data?.default_provider || 'local'
const d = configRes?.data
hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name)
if (!localProvider.value || localProvider.value === '') {
localProvider.value = defaultProvider.value
emit('update:storageProvider', localProvider.value)
}
} catch {
engineStatus.value = []
} finally {
loading.value = false
}
}
watch(() => props.storageProvider, (v) => {
localProvider.value = v || defaultProvider.value || 'local'
}, { immediate: true })
onMounted(load)
</script>
<style lang="less" scoped>
.kb-storage-settings {
width: 100%;
}
.section-header {
margin-bottom: 32px;
h2 {
font-size: 20px;
font-weight: 600;
color: #333;
margin: 0 0 8px 0;
}
.section-description {
font-size: 14px;
color: #666;
margin: 0;
line-height: 1.5;
}
}
.loading-inline {
display: flex;
align-items: center;
gap: 8px;
padding: 16px 0;
}
.settings-group {
display: flex;
flex-direction: column;
}
.setting-row {
display: flex;
align-items: flex-start;
justify-content: space-between;
padding: 20px 0;
border-bottom: 1px solid #e5e7eb;
}
.setting-info {
flex: 1;
max-width: 65%;
padding-right: 24px;
label {
font-size: 15px;
font-weight: 500;
color: #333;
display: block;
margin-bottom: 4px;
}
.desc {
font-size: 13px;
color: #666;
margin: 0;
line-height: 1.5;
}
}
.setting-control {
flex-shrink: 0;
min-width: 280px;
display: flex;
flex-direction: column;
align-items: flex-start;
gap: 6px;
}
.select-option {
display: inline-flex;
align-items: center;
gap: 8px;
}
.option-hint {
font-size: 12px;
color: #999;
margin: 0;
line-height: 1.4;
}
.go-settings {
font-size: 13px;
color: var(--td-brand-color, #0052d9);
margin-top: 8px;
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
</style>

View File

@ -3,7 +3,7 @@
<Menu></Menu>
<RouterView />
<div class="upload-mask" v-show="ismask">
<input type="file" style="display: none" ref="uploadInput" accept=".pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xls,.xlsx" />
<input type="file" style="display: none" ref="uploadInput" accept=".pdf,.docx,.doc,.pptx,.ppt,.txt,.md,.jpg,.jpeg,.png,.csv,.xls,.xlsx" />
<UploadMask></UploadMask>
</div>
<!-- 全局设置模态框,供所有 platform 子路由使用 -->

View File

@ -0,0 +1,610 @@
<template>
<div class="parser-engine-settings">
<div class="section-header">
<h2>解析引擎</h2>
<p class="section-description">
文档解析引擎状态及配置。此处设置优先于服务端环境变量,留空则使用环境变量默认值。
</p>
</div>
<div v-if="loading" class="loading-state">
<t-loading size="small" />
<span>加载中...</span>
</div>
<div v-else-if="error" class="error-inline">
<t-alert theme="error" :message="error">
<template #operation>
<t-button size="small" @click="loadAll">重试</t-button>
</template>
</t-alert>
</div>
<template v-else>
<!-- 引擎列表 -->
<div class="engine-section">
<div v-if="engines.length === 0" class="empty-state">
<p class="empty-text">未检测到解析引擎,请确认 DocReader 服务正常运行。</p>
</div>
<div v-else class="engine-grid">
<!-- DocReader 未连接时显示占位卡片 -->
<div v-if="!hasBuiltinEngine" class="engine-card unavailable">
<div class="card-header">
<span class="card-name">builtin</span>
<t-tag theme="danger" variant="light" size="small">未连接</t-tag>
</div>
<p class="card-desc">DocReader 内置解析引擎(docx/pdf/xlsx 等复杂格式)</p>
<div class="docreader-inline">
<div class="status-line">
<t-tag theme="danger" variant="light" size="small">未连接</t-tag>
<t-tag theme="default" variant="light" size="small">{{ docreaderTransport === 'http' ? 'HTTP' : 'gRPC' }}</t-tag>
<span v-if="docreaderAddrEnv" class="env-hint">当前: {{ docreaderAddrEnv }}</span>
</div>
<p class="desc docreader-desc">
修改请设置环境变量 <code>DOCREADER_ADDR</code>、<code>DOCREADER_TRANSPORT</code>(grpc/http),重启服务生效。
</p>
</div>
</div>
<div
v-for="engine in sortedEngines"
:key="engine.Name"
:class="['engine-card', { unavailable: !engine.Available }]"
>
<div class="card-header">
<span class="card-name">{{ engine.Name }}</span>
<t-tag v-if="engine.Available" theme="success" variant="light" size="small">可用</t-tag>
<t-tooltip v-else-if="engine.UnavailableReason" :content="engine.UnavailableReason" placement="top">
<t-tag theme="danger" variant="light" size="small" class="tag-with-tooltip">不可用</t-tag>
</t-tooltip>
<t-tag v-else theme="danger" variant="light" size="small">不可用</t-tag>
<a
v-if="engineDocLink(engine.Name)"
:href="engineDocLink(engine.Name)"
target="_blank"
rel="noopener noreferrer"
class="card-doc-link"
>{{ engineDocLabel(engine.Name) }} ↗</a>
</div>
<p class="card-desc">{{ engine.Description }}</p>
<!-- builtin 引擎:内嵌 DocReader 连接信息 -->
<div v-if="engine.Name === 'builtin'" class="docreader-inline">
<div class="status-line">
<t-tag v-if="connected" theme="success" variant="light" size="small">已连接</t-tag>
<t-tag v-else theme="danger" variant="light" size="small">未连接</t-tag>
<t-tag theme="default" variant="light" size="small">{{ docreaderTransport === 'http' ? 'HTTP' : 'gRPC' }}</t-tag>
<span v-if="docreaderAddrEnv" class="env-hint">当前: {{ docreaderAddrEnv }}</span>
</div>
<p class="desc docreader-desc">
修改请设置环境变量 <code>DOCREADER_ADDR</code>、<code>DOCREADER_TRANSPORT</code>(grpc/http),重启服务生效。
</p>
</div>
<div class="card-types">
<t-tag
v-for="ft in (engine.FileTypes || [])"
:key="ft"
size="small"
variant="light"
theme="default"
>{{ ft }}</t-tag>
</div>
<!-- mineru 自建配置 -->
<div v-if="engine.Name === 'mineru'" class="card-config">
<div class="config-item">
<label>自建端点</label>
<t-input
v-model="config.mineru_endpoint"
size="small"
placeholder="如 https://your-mineru.example.com"
clearable
/>
</div>
<div class="config-item">
<label>Backend</label>
<t-select v-model="config.mineru_model" size="small" placeholder="默认 pipeline" clearable>
<t-option value="pipeline" label="pipeline" />
<t-option value="vlm-auto-engine" label="vlm-auto-engine" />
<t-option value="vlm-http-client" label="vlm-http-client" />
<t-option value="hybrid-auto-engine" label="hybrid-auto-engine" />
<t-option value="hybrid-http-client" label="hybrid-http-client" />
</t-select>
</div>
<div class="config-toggles">
<t-checkbox v-model="config.mineru_enable_formula">公式识别</t-checkbox>
<t-checkbox v-model="config.mineru_enable_table">表格识别</t-checkbox>
<t-checkbox v-model="config.mineru_enable_ocr">OCR</t-checkbox>
</div>
<div class="config-item">
<label>语言</label>
<t-input
v-model="config.mineru_language"
size="small"
placeholder="如 ch、en、ja(默认 ch)"
clearable
/>
</div>
</div>
<!-- mineru_cloud 云 API 配置 -->
<div v-if="engine.Name === 'mineru_cloud'" class="card-config">
<div class="config-item">
<label>API Key</label>
<t-input
v-model="config.mineru_api_key"
size="small"
type="password"
placeholder="MinerU 云服务 API Key"
clearable
/>
</div>
<div class="config-item">
<label>API 地址</label>
<t-input
v-model="config.mineru_api_base_url"
size="small"
placeholder="如 https://api.mineru.net"
clearable
/>
</div>
<div class="config-item">
<label>Model Version</label>
<t-select v-model="config.mineru_cloud_model" size="small" placeholder="默认 pipeline" clearable>
<t-option value="pipeline" label="pipeline" />
<t-option value="vlm" label="vlm(视觉语言模型)" />
<t-option value="MinerU-HTML" label="MinerU-HTML(HTML 解析)" />
</t-select>
</div>
<div class="config-toggles">
<t-checkbox v-model="config.mineru_cloud_enable_formula">公式识别</t-checkbox>
<t-checkbox v-model="config.mineru_cloud_enable_table">表格识别</t-checkbox>
<t-checkbox v-model="config.mineru_cloud_enable_ocr">OCR</t-checkbox>
</div>
<div class="config-item">
<label>语言</label>
<t-input
v-model="config.mineru_cloud_language"
size="small"
placeholder="如 ch、en、ja(默认 ch)"
clearable
/>
</div>
</div>
</div>
</div>
</div>
<!-- 检测与保存 -->
<div class="save-bar">
<t-button theme="default" variant="outline" :loading="checking" @click="onCheck">
使用当前参数检测
</t-button>
<t-button theme="primary" :loading="saving" @click="onSave">保存配置</t-button>
<span v-if="checkMessage" class="save-msg hint">{{ checkMessage }}</span>
<span v-else-if="saveMessage" :class="['save-msg', saveSuccess ? 'success' : 'error']">
{{ saveMessage }}
</span>
</div>
</template>
</div>
</template>
<script setup lang="ts">
import { ref, computed, onMounted } from 'vue'
import {
getParserEngines,
getParserEngineConfig,
updateParserEngineConfig,
checkParserEngines,
type ParserEngineInfo,
type ParserEngineConfig,
} from '@/api/system'
const CONFIGURABLE_ENGINES = new Set(['mineru', 'mineru_cloud'])
/** 各解析引擎的项目/官方文档地址 */
const ENGINE_DOC_LINKS: Record<string, { url: string; label: string }> = {
markitdown: { url: 'https://github.com/microsoft/markitdown', label: 'Markitdown 文档' },
mineru: { url: 'https://github.com/opendatalab/MinerU', label: 'MinerU 文档' },
mineru_cloud: { url: 'https://mineru.net/apiManage/docs', label: 'MinerU 文档' },
}
/** 解析引擎配置默认值(与 DocReader/Python 侧一致) */
const DEFAULT_PARSER_CONFIG: ParserEngineConfig = {
docreader_addr: '',
docreader_transport: 'grpc',
mineru_endpoint: '',
mineru_api_key: '',
mineru_api_base_url: 'https://mineru.net/api/v4',
mineru_model: 'pipeline',
mineru_enable_formula: true,
mineru_enable_table: true,
mineru_enable_ocr: true,
mineru_language: 'ch',
mineru_cloud_model: 'pipeline',
mineru_cloud_enable_formula: true,
mineru_cloud_enable_table: true,
mineru_cloud_enable_ocr: true,
mineru_cloud_language: 'ch',
}
const engines = ref<ParserEngineInfo[]>([])
const docreaderAddrEnv = ref('')
const docreaderTransport = ref<'grpc' | 'http'>('grpc')
const connected = ref(false)
const loading = ref(true)
const error = ref('')
const config = ref<ParserEngineConfig>({ ...DEFAULT_PARSER_CONFIG })
const saving = ref(false)
const saveMessage = ref('')
const saveSuccess = ref(false)
const checking = ref(false)
const checkMessage = ref('')
const hasBuiltinEngine = computed(() => engines.value.some(e => e.Name === 'builtin'))
/** 固定展示顺序,未列出的引擎排在末尾按名称排序 */
const ENGINE_ORDER: Record<string, number> = {
builtin: 0,
simple: 1,
markitdown: 2,
mineru: 3,
mineru_cloud: 4,
}
const sortedEngines = computed(() => {
return [...engines.value].sort((a, b) => {
const oa = ENGINE_ORDER[a.Name] ?? 100
const ob = ENGINE_ORDER[b.Name] ?? 100
if (oa !== ob) return oa - ob
return a.Name.localeCompare(b.Name)
})
})
function hasConfigFields(engineName: string): boolean {
return CONFIGURABLE_ENGINES.has(engineName)
}
function engineDocLink(name: string): string | undefined {
return ENGINE_DOC_LINKS[name]?.url
}
function engineDocLabel(name: string): string {
return ENGINE_DOC_LINKS[name]?.label ?? '文档'
}
async function loadEngines() {
try {
const res = await getParserEngines()
engines.value = res?.data ?? []
docreaderAddrEnv.value = res?.docreader_addr ?? ''
const t = (res?.docreader_transport ?? 'grpc').toLowerCase()
docreaderTransport.value = t === 'http' ? 'http' : 'grpc'
connected.value = res?.connected ?? (engines.value.length > 0)
} catch (e: any) {
error.value = e?.message || '加载解析引擎列表失败'
engines.value = []
connected.value = false
}
}
async function loadConfig() {
try {
const res = await getParserEngineConfig()
const data = res?.data
config.value = {
docreader_addr: data?.docreader_addr ?? DEFAULT_PARSER_CONFIG.docreader_addr ?? '',
docreader_transport: data?.docreader_transport ?? DEFAULT_PARSER_CONFIG.docreader_transport ?? 'grpc',
mineru_endpoint: data?.mineru_endpoint ?? DEFAULT_PARSER_CONFIG.mineru_endpoint ?? '',
mineru_api_key: data?.mineru_api_key ?? DEFAULT_PARSER_CONFIG.mineru_api_key ?? '',
mineru_api_base_url: data?.mineru_api_base_url ?? DEFAULT_PARSER_CONFIG.mineru_api_base_url ?? '',
mineru_model: data?.mineru_model ?? DEFAULT_PARSER_CONFIG.mineru_model ?? '',
mineru_enable_formula: data?.mineru_enable_formula ?? DEFAULT_PARSER_CONFIG.mineru_enable_formula ?? true,
mineru_enable_table: data?.mineru_enable_table ?? DEFAULT_PARSER_CONFIG.mineru_enable_table ?? true,
mineru_enable_ocr: data?.mineru_enable_ocr ?? DEFAULT_PARSER_CONFIG.mineru_enable_ocr ?? true,
mineru_language: data?.mineru_language ?? DEFAULT_PARSER_CONFIG.mineru_language ?? 'ch',
mineru_cloud_model: data?.mineru_cloud_model ?? DEFAULT_PARSER_CONFIG.mineru_cloud_model ?? '',
mineru_cloud_enable_formula: data?.mineru_cloud_enable_formula ?? DEFAULT_PARSER_CONFIG.mineru_cloud_enable_formula ?? true,
mineru_cloud_enable_table: data?.mineru_cloud_enable_table ?? DEFAULT_PARSER_CONFIG.mineru_cloud_enable_table ?? true,
mineru_cloud_enable_ocr: data?.mineru_cloud_enable_ocr ?? DEFAULT_PARSER_CONFIG.mineru_cloud_enable_ocr ?? true,
mineru_cloud_language: data?.mineru_cloud_language ?? DEFAULT_PARSER_CONFIG.mineru_cloud_language ?? 'ch',
}
} catch {
config.value = { ...DEFAULT_PARSER_CONFIG }
}
}
async function loadAll() {
loading.value = true
error.value = ''
await Promise.all([loadEngines(), loadConfig()])
loading.value = false
}
function buildConfigPayload(): ParserEngineConfig {
return {
docreader_addr: config.value.docreader_addr?.trim() ?? '',
docreader_transport: (config.value.docreader_transport ?? 'grpc').trim() || 'grpc',
mineru_endpoint: config.value.mineru_endpoint?.trim() ?? '',
mineru_api_key: config.value.mineru_api_key?.trim() ?? '',
mineru_api_base_url: config.value.mineru_api_base_url?.trim() ?? '',
mineru_model: config.value.mineru_model?.trim() ?? '',
mineru_enable_formula: config.value.mineru_enable_formula,
mineru_enable_table: config.value.mineru_enable_table,
mineru_enable_ocr: config.value.mineru_enable_ocr,
mineru_language: config.value.mineru_language?.trim() ?? '',
mineru_cloud_model: config.value.mineru_cloud_model?.trim() ?? '',
mineru_cloud_enable_formula: config.value.mineru_cloud_enable_formula,
mineru_cloud_enable_table: config.value.mineru_cloud_enable_table,
mineru_cloud_enable_ocr: config.value.mineru_cloud_enable_ocr,
mineru_cloud_language: config.value.mineru_cloud_language?.trim() ?? '',
}
}
async function onCheck() {
if (!connected) {
checkMessage.value = '请先确保 DocReader 服务已通过环境变量配置并已连接'
return
}
checking.value = true
checkMessage.value = ''
try {
const res = await checkParserEngines(buildConfigPayload())
engines.value = res?.data ?? []
checkMessage.value = '已使用当前填写参数检测,上方状态已更新'
setTimeout(() => { checkMessage.value = '' }, 3000)
} catch (e: any) {
checkMessage.value = e?.message || '检测失败'
} finally {
checking.value = false
}
}
async function onSave() {
saving.value = true
saveMessage.value = ''
try {
await updateParserEngineConfig(buildConfigPayload())
saveSuccess.value = true
saveMessage.value = '保存成功'
loadEngines()
} catch (e: any) {
saveSuccess.value = false
saveMessage.value = e?.message || '保存失败'
} finally {
saving.value = false
}
}
onMounted(loadAll)
</script>
<style lang="less" scoped>
.parser-engine-settings {
width: 100%;
}
// ---- 页面标题 ----
.section-header {
margin-bottom: 32px;
h2 {
font-size: 20px;
font-weight: 600;
color: #333;
margin: 0 0 8px 0;
}
.section-description {
font-size: 14px;
color: #666;
margin: 0;
line-height: 1.5;
}
}
// ---- 加载 / 错误 ----
.loading-state {
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
padding: 48px 0;
color: #999;
font-size: 14px;
}
.error-inline {
padding: 16px 0;
}
.docreader-inline {
padding: 10px 12px;
background: #f7f8fa;
border-radius: 6px;
.status-line {
margin-bottom: 6px;
}
}
.docreader-desc {
margin: 0;
font-size: 12px;
color: #888;
line-height: 1.6;
code {
padding: 1px 5px;
font-size: 11px;
background: #eee;
border-radius: 3px;
}
}
.card-doc-link {
margin-left: auto;
font-size: 12px;
color: #999;
text-decoration: none;
white-space: nowrap;
&:hover {
color: var(--td-brand-color, #0052d9);
}
}
.status-line {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
}
.env-hint {
font-size: 12px;
color: #999;
}
// ---- 引擎列表 ----
.engine-section {
margin-top: 0;
}
.engine-grid {
columns: 2;
column-gap: 12px;
@media (max-width: 860px) {
columns: 1;
}
}
.engine-card {
break-inside: avoid;
margin-bottom: 12px;
padding: 16px;
border: 1px solid #e5e7eb;
border-radius: 8px;
background: #fff;
transition: border-color 0.2s;
display: flex;
flex-direction: column;
gap: 8px;
&:hover {
border-color: #c0c4cc;
}
&.unavailable {
background: #fafafa;
}
}
.card-header {
display: flex;
align-items: center;
gap: 8px;
}
.card-name {
font-size: 15px;
font-weight: 600;
color: #333;
font-family: 'SF Mono', 'Monaco', 'Menlo', monospace;
}
.card-desc {
font-size: 13px;
color: #666;
margin: 0;
line-height: 1.5;
}
.card-types {
display: flex;
flex-wrap: wrap;
gap: 4px;
}
.card-reason {
font-size: 12px;
color: #e34d59;
margin: 0;
}
// ---- 卡片内配置区 ----
.card-config {
margin-top: 4px;
padding-top: 12px;
border-top: 1px dashed #e5e7eb;
display: flex;
flex-direction: column;
gap: 10px;
}
.config-item {
display: flex;
flex-direction: column;
gap: 4px;
label {
font-size: 12px;
font-weight: 500;
color: #555;
}
}
.config-toggles {
display: flex;
flex-wrap: wrap;
gap: 12px;
}
// ---- 空状态 ----
.empty-state {
padding: 48px 0;
text-align: center;
.empty-text {
font-size: 14px;
color: #999;
margin: 0;
}
}
// ---- 保存栏 ----
.save-bar {
display: flex;
align-items: center;
gap: 12px;
margin-top: 24px;
padding-top: 24px;
border-top: 1px solid #e5e7eb;
}
.save-msg {
font-size: 13px;
&.success {
color: #52c41a;
}
&.error {
color: #ff4d4f;
}
&.hint {
color: #666;
}
}
.tag-with-tooltip {
cursor: help;
}
</style>

View File

@ -94,6 +94,16 @@
<WebSearchSettings />
</div>
<!-- 解析引擎 -->
<div v-if="currentSection === 'parser'" class="section">
<ParserEngineSettings />
</div>
<!-- 存储引擎 -->
<div v-if="currentSection === 'storage'" class="section">
<StorageEngineSettings />
</div>
<!-- 系统信息 -->
<div v-if="currentSection === 'system'" class="section">
<SystemInfo />
@ -135,6 +145,8 @@ import ModelSettings from './ModelSettings.vue'
import OllamaSettings from './OllamaSettings.vue'
import McpSettings from './McpSettings.vue'
import WebSearchSettings from './WebSearchSettings.vue'
import ParserEngineSettings from './ParserEngineSettings.vue'
import StorageEngineSettings from './StorageEngineSettings.vue'
const route = useRoute()
const router = useRouter()
@ -160,6 +172,8 @@ const navItems = computed(() => [
},
{ key: 'ollama', icon: 'server', label: 'Ollama' },
{ key: 'websearch', icon: 'search', label: t('settings.webSearchConfig') },
{ key: 'parser', icon: 'file-search', label: '解析引擎' },
{ key: 'storage', icon: 'cloud', label: t('settings.storageEngine') },
{ key: 'mcp', icon: 'tools', label: t('settings.mcpService') },
{ key: 'system', icon: 'info-circle', label: t('settings.systemSettings') },
{ key: 'tenant', icon: 'user-circle', label: t('settings.tenantInfo') },

View File

@ -0,0 +1,739 @@
<template>
<div class="storage-engine-settings">
<div class="section-header">
<h2>存储引擎</h2>
<p class="section-description">
配置文档与图片的存储方式。此处设置各引擎参数,知识库中仅选择使用哪个引擎。存储引擎会影响文档上传存储以及文档内图片的存储。
</p>
</div>
<div v-if="loading" class="loading-state">
<t-loading size="small" />
<span>加载中...</span>
</div>
<div v-else-if="error" class="error-inline">
<t-alert theme="error" :message="error">
<template #operation>
<t-button size="small" @click="loadAll">重试</t-button>
</template>
</t-alert>
</div>
<template v-else>
<div class="engine-section">
<div class="section-title-row">
<div class="section-title-info">
<h3>引擎配置</h3>
<p>为每种存储引擎配置参数,知识库创建或编辑时选择使用哪个引擎。</p>
</div>
</div>
<div class="engine-list">
<!-- Local -->
<div class="engine-block">
<div class="block-header">
<span class="block-name">Local(本地存储)</span>
<t-tag theme="success" variant="light" size="small">可用</t-tag>
</div>
<p class="block-desc">使用服务器本地文件系统存储文件,仅适合单机部署。</p>
<div class="block-config">
<div class="config-item">
<label>路径前缀(可选)</label>
<t-input
v-model="config.local.path_prefix"
size="small"
placeholder="如 weknora/images"
clearable
/>
</div>
</div>
</div>
<!-- MinIO -->
<div class="engine-block">
<div class="block-header">
<span class="block-name">MinIO</span>
<t-tag v-if="minioAvailable" theme="success" variant="light" size="small">可用</t-tag>
<t-tag v-else theme="default" variant="light" size="small">需要配置</t-tag>
</div>
<p class="block-desc">S3 兼容的自托管对象存储,适合内网和私有云部署。</p>
<div class="minio-mode-tabs">
<div
:class="['mode-tab', { active: config.minio.mode !== 'remote' }]"
@click="config.minio.mode = 'docker'"
>
<span class="tab-label">Docker 部署</span>
<t-tag v-if="minioEnvAvailable" theme="success" variant="light" size="small">已检测</t-tag>
<t-tag v-else theme="default" variant="light" size="small">未检测到</t-tag>
</div>
<div
:class="['mode-tab', { active: config.minio.mode === 'remote' }]"
@click="config.minio.mode = 'remote'"
>
<span class="tab-label">远程 MinIO</span>
</div>
</div>
<!-- Docker mode -->
<div v-if="config.minio.mode !== 'remote'" class="minio-mode-content">
<p v-if="minioEnvAvailable" class="mode-hint success">
已检测到 Docker 部署的 MinIO 环境变量,连接信息(Endpoint、Access Key)由环境变量提供,无需手动填写。
</p>
<p v-else class="mode-hint warning">
未检测到 MinIO 环境变量(MINIO_ENDPOINT、MINIO_ACCESS_KEY_ID、MINIO_SECRET_ACCESS_KEY),请确认 Docker Compose 配置正确。
</p>
<div class="block-config">
<div class="config-item">
<label>Bucket 名称</label>
<t-select
v-model="config.minio.bucket_name"
size="small"
filterable
creatable
placeholder="选择或输入 Bucket"
:loading="loadingBuckets"
:disabled="!minioEnvAvailable"
@focus="loadMinioBuckets"
style="width: 100%;"
>
<t-option
v-for="b in minioBuckets"
:key="b.name"
:value="b.name"
:label="b.name"
/>
</t-select>
</div>
<div class="config-item config-item--inline">
<label>Use SSL</label>
<t-switch v-model="config.minio.use_ssl" size="small" />
</div>
<div class="config-item">
<label>路径前缀(可选)</label>
<t-input
v-model="config.minio.path_prefix"
size="small"
placeholder="如 weknora"
clearable
/>
</div>
</div>
<div v-if="minioEnvAvailable" class="check-bar">
<t-button size="small" variant="outline" :loading="checkingMinio" @click="onCheckMinio">测试连接</t-button>
<span v-if="minioCheckResult" :class="['check-msg', minioCheckResult.ok ? 'success' : 'error']">
{{ minioCheckResult.message }}
</span>
</div>
</div>
<!-- Remote mode -->
<div v-else class="minio-mode-content">
<p class="mode-hint">连接到远程 MinIO 服务,需要手动填写连接信息。</p>
<div class="block-config">
<div class="config-item">
<label>Endpoint</label>
<t-input
v-model="config.minio.endpoint"
size="small"
placeholder="如 minio.example.com:9000"
clearable
/>
</div>
<div class="config-item">
<label>Access Key ID</label>
<t-input
v-model="config.minio.access_key_id"
size="small"
placeholder="MinIO Access Key"
clearable
/>
</div>
<div class="config-item">
<label>Secret Access Key</label>
<t-input
v-model="config.minio.secret_access_key"
size="small"
type="password"
placeholder="MinIO Secret Key"
clearable
/>
</div>
<div class="config-item">
<label>Bucket 名称</label>
<t-input
v-model="config.minio.bucket_name"
size="small"
placeholder="存储桶名称"
clearable
/>
</div>
<div class="config-item config-item--inline">
<label>Use SSL</label>
<t-switch v-model="config.minio.use_ssl" size="small" />
</div>
<div class="config-item">
<label>路径前缀(可选)</label>
<t-input
v-model="config.minio.path_prefix"
size="small"
placeholder="如 weknora"
clearable
/>
</div>
</div>
<div class="check-bar">
<t-button size="small" variant="outline" :loading="checkingMinio" @click="onCheckMinio">测试连接</t-button>
<span v-if="minioCheckResult" :class="['check-msg', minioCheckResult.ok ? 'success' : 'error']">
{{ minioCheckResult.message }}
</span>
</div>
</div>
</div>
<!-- COS -->
<div class="engine-block">
<div class="block-header">
<span class="block-name">腾讯云 COS</span>
<t-tag theme="success" variant="light" size="small">可配置</t-tag>
</div>
<p class="block-desc">腾讯云对象存储服务,适合公有云部署,支持 CDN 加速。</p>
<div class="block-config">
<div class="config-item">
<label>Secret ID</label>
<t-input
v-model="config.cos.secret_id"
size="small"
placeholder="腾讯云 API 密钥 SecretId"
clearable
/>
</div>
<div class="config-item">
<label>Secret Key</label>
<t-input
v-model="config.cos.secret_key"
size="small"
type="password"
placeholder="腾讯云 API 密钥 SecretKey"
clearable
/>
</div>
<div class="config-item">
<label>Region</label>
<t-input
v-model="config.cos.region"
size="small"
placeholder="如 ap-guangzhou"
clearable
/>
</div>
<div class="config-item">
<label>Bucket 名称</label>
<t-input
v-model="config.cos.bucket_name"
size="small"
placeholder="存储桶名称"
clearable
/>
</div>
<div class="config-item">
<label>App ID</label>
<t-input
v-model="config.cos.app_id"
size="small"
placeholder="腾讯云账号 AppID"
clearable
/>
</div>
<div class="config-item">
<label>路径前缀(可选)</label>
<t-input
v-model="config.cos.path_prefix"
size="small"
placeholder="如 weknora"
clearable
/>
</div>
</div>
<div class="check-bar">
<t-button size="small" variant="outline" :loading="checkingCos" @click="onCheckCos">测试连接</t-button>
<span v-if="cosCheckResult" :class="['check-msg', cosCheckResult.ok ? 'success' : 'error']">
{{ cosCheckResult.message }}
</span>
</div>
</div>
</div>
</div>
<div class="default-and-save">
<div class="default-row">
<label>默认引擎</label>
<t-select
v-model="config.default_provider"
size="small"
style="width: 200px;"
>
<t-option value="local" label="Local(本地)" />
<t-option value="minio" label="MinIO" />
<t-option value="cos" label="腾讯云 COS" />
</t-select>
<span class="hint">新建知识库时默认选用的存储引擎</span>
</div>
<div class="save-bar">
<t-button theme="primary" :loading="saving" @click="onSave">保存配置</t-button>
<span v-if="saveMessage" :class="['save-msg', saveSuccess ? 'success' : 'error']">
{{ saveMessage }}
</span>
</div>
</div>
</template>
</div>
</template>
<script setup lang="ts">
import { ref, computed, onMounted } from 'vue'
import {
getStorageEngineConfig,
updateStorageEngineConfig,
getStorageEngineStatus,
listMinioBuckets,
checkStorageEngine,
type StorageEngineConfig,
type MinioBucketInfo,
} from '@/api/system'
const defaultConfig = (): StorageEngineConfig => ({
default_provider: 'local',
local: { path_prefix: '' },
minio: { mode: 'docker', endpoint: '', access_key_id: '', secret_access_key: '', bucket_name: '', use_ssl: false, path_prefix: '' },
cos: {
secret_id: '',
secret_key: '',
region: '',
bucket_name: '',
app_id: '',
path_prefix: '',
},
})
const loading = ref(true)
const error = ref('')
const config = ref<StorageEngineConfig>(defaultConfig())
const engineStatus = ref<{ local: boolean; minio: boolean; cos: boolean }>({
local: true,
minio: false,
cos: true,
})
const minioEnvAvailable = ref(false)
const minioBuckets = ref<MinioBucketInfo[]>([])
const loadingBuckets = ref(false)
const saving = ref(false)
const saveMessage = ref('')
const saveSuccess = ref(false)
const checkingMinio = ref(false)
const minioCheckResult = ref<{ ok: boolean; message: string } | null>(null)
const checkingCos = ref(false)
const cosCheckResult = ref<{ ok: boolean; message: string } | null>(null)
const minioAvailable = computed(() => {
if (config.value.minio?.mode === 'remote') {
return !!(config.value.minio.endpoint && config.value.minio.access_key_id && config.value.minio.secret_access_key)
}
return minioEnvAvailable.value
})
async function loadConfig() {
try {
const res = await getStorageEngineConfig()
const d = res?.data
if (d) {
config.value = {
default_provider: d.default_provider || 'local',
local: d.local ? { path_prefix: d.local.path_prefix || '' } : { path_prefix: '' },
minio: d.minio
? {
mode: d.minio.mode || 'docker',
endpoint: d.minio.endpoint || '',
access_key_id: d.minio.access_key_id || '',
secret_access_key: d.minio.secret_access_key || '',
bucket_name: d.minio.bucket_name || '',
use_ssl: d.minio.use_ssl ?? false,
path_prefix: d.minio.path_prefix || '',
}
: defaultConfig().minio!,
cos: d.cos
? {
secret_id: d.cos.secret_id || '',
secret_key: d.cos.secret_key || '',
region: d.cos.region || '',
bucket_name: d.cos.bucket_name || '',
app_id: d.cos.app_id || '',
path_prefix: d.cos.path_prefix || '',
}
: defaultConfig().cos!,
}
}
} catch {
config.value = defaultConfig()
}
}
async function loadStatus() {
try {
const res = await getStorageEngineStatus()
const engines = res?.data?.engines ?? []
const status = { local: true, minio: false, cos: true }
for (const e of engines) {
if (e.name === 'local') status.local = e.available
if (e.name === 'minio') status.minio = e.available
if (e.name === 'cos') status.cos = e.available
}
engineStatus.value = status
minioEnvAvailable.value = res?.data?.minio_env_available ?? false
} catch {
engineStatus.value = { local: true, minio: false, cos: true }
minioEnvAvailable.value = false
}
}
async function loadMinioBuckets() {
if (!minioEnvAvailable.value || loadingBuckets.value) return
loadingBuckets.value = true
try {
const res = await listMinioBuckets()
if (res?.data?.buckets) {
minioBuckets.value = res.data.buckets
}
} catch {
minioBuckets.value = []
} finally {
loadingBuckets.value = false
}
}
async function loadAll() {
loading.value = true
error.value = ''
try {
await Promise.all([loadConfig(), loadStatus()])
if (minioEnvAvailable.value) loadMinioBuckets()
} catch (e: unknown) {
error.value = e instanceof Error ? e.message : '加载失败'
} finally {
loading.value = false
}
}
function buildPayload(): StorageEngineConfig {
const mode = config.value.minio?.mode || 'docker'
return {
default_provider: config.value.default_provider || 'local',
local: { path_prefix: (config.value.local?.path_prefix || '').trim() },
minio: {
mode,
endpoint: mode === 'remote' ? (config.value.minio?.endpoint || '').trim() : '',
access_key_id: mode === 'remote' ? (config.value.minio?.access_key_id || '').trim() : '',
secret_access_key: mode === 'remote' ? (config.value.minio?.secret_access_key || '').trim() : '',
bucket_name: (config.value.minio?.bucket_name || '').trim(),
use_ssl: config.value.minio?.use_ssl ?? false,
path_prefix: (config.value.minio?.path_prefix || '').trim(),
},
cos: {
secret_id: (config.value.cos?.secret_id || '').trim(),
secret_key: (config.value.cos?.secret_key || '').trim(),
region: (config.value.cos?.region || '').trim(),
bucket_name: (config.value.cos?.bucket_name || '').trim(),
app_id: (config.value.cos?.app_id || '').trim(),
path_prefix: (config.value.cos?.path_prefix || '').trim(),
},
}
}
async function onSave() {
saving.value = true
saveMessage.value = ''
try {
await updateStorageEngineConfig(buildPayload())
await loadStatus()
saveSuccess.value = true
saveMessage.value = '保存成功'
} catch (e: unknown) {
saveSuccess.value = false
saveMessage.value = e instanceof Error ? e.message : '保存失败'
} finally {
saving.value = false
}
}
async function onCheckMinio() {
checkingMinio.value = true
minioCheckResult.value = null
try {
const payload = buildPayload()
const res = await checkStorageEngine({ provider: 'minio', minio: payload.minio })
minioCheckResult.value = res?.data ?? { ok: false, message: '未知错误' }
} catch (e: unknown) {
minioCheckResult.value = { ok: false, message: e instanceof Error ? e.message : '请求失败' }
} finally {
checkingMinio.value = false
}
}
async function onCheckCos() {
checkingCos.value = true
cosCheckResult.value = null
try {
const payload = buildPayload()
const res = await checkStorageEngine({ provider: 'cos', cos: payload.cos })
cosCheckResult.value = res?.data ?? { ok: false, message: '未知错误' }
} catch (e: unknown) {
cosCheckResult.value = { ok: false, message: e instanceof Error ? e.message : '请求失败' }
} finally {
checkingCos.value = false
}
}
onMounted(loadAll)
</script>
<style lang="less" scoped>
.storage-engine-settings {
width: 100%;
}
.section-header {
margin-bottom: 32px;
h2 {
font-size: 20px;
font-weight: 600;
color: #333;
margin: 0 0 8px 0;
}
.section-description {
font-size: 14px;
color: #666;
margin: 0;
line-height: 1.5;
}
}
.loading-state,
.error-inline {
padding: 24px 0;
}
.loading-state {
display: flex;
align-items: center;
gap: 8px;
color: #999;
}
.engine-section {
margin-top: 0;
}
.section-title-row {
margin-bottom: 20px;
h3 {
font-size: 17px;
font-weight: 600;
color: #333;
margin: 0 0 6px 0;
}
p {
font-size: 13px;
color: #999;
margin: 0;
line-height: 1.5;
}
}
.engine-list {
display: flex;
flex-direction: column;
gap: 20px;
}
.engine-block {
padding: 20px;
border: 1px solid #e5e7eb;
border-radius: 10px;
background: #fff;
display: flex;
flex-direction: column;
gap: 10px;
}
.block-header {
display: flex;
align-items: center;
gap: 10px;
}
.block-name {
font-size: 16px;
font-weight: 600;
color: #333;
}
.block-desc {
font-size: 13px;
color: #666;
margin: 0;
line-height: 1.5;
}
.block-config {
margin-top: 6px;
padding-top: 14px;
border-top: 1px dashed #e5e7eb;
display: grid;
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
gap: 14px 24px;
}
.config-item {
display: flex;
flex-direction: column;
gap: 4px;
label {
font-size: 12px;
font-weight: 500;
color: #555;
}
&--inline {
flex-direction: row;
align-items: center;
gap: 10px;
label {
flex-shrink: 0;
}
}
}
// ---- MinIO mode tabs ----
.minio-mode-tabs {
display: flex;
gap: 8px;
margin-top: 4px;
}
.mode-tab {
display: flex;
align-items: center;
gap: 6px;
padding: 8px 16px;
border: 1px solid #e5e7eb;
border-radius: 8px;
cursor: pointer;
transition: all 0.15s;
background: #fafafa;
&:hover {
border-color: #c0c4cc;
}
&.active {
border-color: var(--td-brand-color, #0052d9);
background: var(--td-brand-color-light, #f0f5ff);
}
.tab-label {
font-size: 13px;
font-weight: 500;
color: #333;
}
}
.minio-mode-content {
margin-top: 4px;
}
.mode-hint {
font-size: 13px;
color: #666;
margin: 0 0 4px 0;
line-height: 1.5;
&.success {
color: #52c41a;
}
&.warning {
color: #e6a23c;
}
}
.default-and-save {
margin-top: 24px;
padding-top: 24px;
border-top: 1px solid #e5e7eb;
}
.default-row {
display: flex;
align-items: center;
gap: 12px;
margin-bottom: 16px;
label {
font-size: 14px;
font-weight: 500;
color: #333;
}
.hint {
font-size: 13px;
color: #999;
}
}
.save-bar {
display: flex;
align-items: center;
gap: 12px;
}
.save-msg {
font-size: 13px;
&.success {
color: #52c41a;
}
&.error {
color: #ff4d4f;
}
}
.check-bar {
display: flex;
align-items: center;
gap: 10px;
margin-top: 12px;
padding-top: 12px;
border-top: 1px dashed #f0f0f0;
}
.check-msg {
font-size: 13px;
&.success {
color: #52c41a;
}
&.error {
color: #ff4d4f;
}
}
</style>

View File

@ -31,6 +31,13 @@
<div class="setting-control">
<span class="info-value">
{{ systemInfo?.version || $t('system.unknown') }}
<t-tag
v-if="systemInfo?.edition"
:theme="systemInfo.edition === 'lite' ? 'primary' : 'default'"
variant="light"
size="small"
style="margin-left: 8px;"
>{{ systemInfo.edition === 'lite' ? 'Lite' : 'Standard' }}</t-tag>
<span v-if="systemInfo?.commit_id" class="commit-info">
({{ systemInfo.commit_id }})
</span>

View File

@ -22,6 +22,11 @@ export default defineConfig({
target: 'http://localhost:8080',
changeOrigin: true,
secure: false,
},
'/files': {
target: 'http://localhost:8080',
changeOrigin: true,
secure: false,
}
}
}

11
go.mod
View File

@ -3,7 +3,9 @@ module github.com/Tencent/WeKnora
go 1.24.11
require (
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0
github.com/PuerkitoBio/goquery v1.10.3
github.com/asg017/sqlite-vec-go-bindings v0.1.6
github.com/chromedp/chromedp v0.14.2
github.com/duckdb/duckdb-go/v2 v2.5.4
github.com/elastic/go-elasticsearch/v7 v7.17.10
@ -12,7 +14,7 @@ require (
github.com/gin-gonic/gin v1.11.0
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/golang-jwt/jwt/v5 v5.3.0
github.com/golang-migrate/migrate/v4 v4.19.0
github.com/golang-migrate/migrate/v4 v4.19.1
github.com/google/jsonschema-go v0.4.2
github.com/google/uuid v1.6.0
github.com/hibiken/asynq v0.25.1
@ -51,13 +53,15 @@ require (
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/postgres v1.5.11
gorm.io/gorm v1.25.12
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.30.0
)
require (
cloud.google.com/go/auth v0.18.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/JohannesKaufmann/dom v0.2.0 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
@ -133,8 +137,6 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/invopop/jsonschema v0.13.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@ -155,6 +157,7 @@ require (
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.8-0.20251223041313-25746c47c1a7 // indirect
github.com/milvus-io/milvus/pkg/v2 v2.6.7-0.20251201120310-af64f2acba38 // indirect
github.com/minio/crc64nvme v1.0.1 // indirect

28
go.sum
View File

@ -1339,6 +1339,10 @@ github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMd
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k=
github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ=
github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo=
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 h1:mklaPbT4f/EiDr1Q+zPrEt9lgKAkVrIBtWf33d9GpVA=
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0/go.mod h1:D56Cl9r8M5i3UwAchE+LlLc5hPN3kJtdZNVJn06lSHU=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
@ -1393,6 +1397,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww=
github.com/asg017/sqlite-vec-go-bindings v0.1.6/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
@ -1721,8 +1727,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@ -1896,14 +1902,10 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVT
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@ -2063,6 +2065,8 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
@ -2253,7 +2257,11 @@ github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoH
github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@ -2416,6 +2424,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
@ -3577,8 +3587,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314=
gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -470,9 +470,10 @@ func (t *KnowledgeSearchTool) concurrentSearchByTargets(
return
}
// Wrap results with metadata
// Wrap results with metadata and write back KB ID
mu.Lock()
for _, r := range kbResults {
r.KnowledgeBaseID = st.KnowledgeBaseID
allResults = append(allResults, &searchResultWithMeta{
SearchResult: r,
SourceQuery: q,

View File

@ -246,11 +246,11 @@ func (r *chunkRepository) ListChunkByParentID(
return chunks, nil
}
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields.
// Note: This will update all fields including metadata and content_hash.
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields
// except SeqID (auto-increment, must not be overwritten).
// Make sure the chunk object is complete (e.g., fetched from DB) before calling this method.
func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error {
return r.db.WithContext(ctx).Save(chunk).Error
return r.db.WithContext(ctx).Omit("SeqID").Save(chunk).Error
}
// UpdateChunks updates chunks in batch using raw SQL for efficiency.
@ -332,23 +332,46 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
args = append(args, id)
}
sql := fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = (CASE %s END)::boolean,
tag_id = CASE %s END,
flags = (CASE %s END)::integer,
status = (CASE %s END)::integer,
updated_at = NOW()
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
strings.Join(inPlaceholders, ","),
)
isPostgres := r.db.Dialector.Name() == "postgres"
var sql string
if isPostgres {
sql = fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = (CASE %s END)::boolean,
tag_id = CASE %s END,
flags = (CASE %s END)::integer,
status = (CASE %s END)::integer,
updated_at = NOW()
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
strings.Join(inPlaceholders, ","),
)
} else {
sql = fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = CASE %s END,
tag_id = CASE %s END,
flags = CASE %s END,
status = CASE %s END,
updated_at = datetime('now')
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
strings.Join(inPlaceholders, ","),
)
}
return r.db.WithContext(ctx).Exec(sql, args...).Error
}

View File

@ -0,0 +1,613 @@
package sqlite
import (
"context"
"fmt"
"strings"
"time"
"unicode"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
"github.com/google/uuid"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// sqliteEmbedding stores metadata alongside the vec0 virtual table rows
type sqliteEmbedding struct {
ID uint `gorm:"primarykey;autoIncrement"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
SourceID string `gorm:"column:source_id;not null;uniqueIndex:idx_sqlite_emb_source"`
SourceType int `gorm:"column:source_type;not null;uniqueIndex:idx_sqlite_emb_source"`
ChunkID string `gorm:"column:chunk_id;index"`
KnowledgeID string `gorm:"column:knowledge_id;index"`
KnowledgeBaseID string `gorm:"column:knowledge_base_id;index"`
TagID string `gorm:"column:tag_id;index"`
Content string `gorm:"column:content;not null"`
Dimension int `gorm:"column:dimension;not null"`
IsEnabled *bool `gorm:"column:is_enabled;default:true;index"`
}
func (sqliteEmbedding) TableName() string { return "lite_embeddings" }
type sqliteRepository struct {
db *gorm.DB
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
}
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
logger.GetLogger(context.Background()).Info("[SQLite] Initializing SQLite retriever engine repository with sqlite-vec")
if err := db.AutoMigrate(&sqliteEmbedding{}); err != nil {
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
}
initFTS5(db)
repo := &sqliteRepository{
db: db,
vecTables: make(map[int]bool),
}
repo.ensureExistingVecTables()
return repo
}
func initFTS5(db *gorm.DB) {
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
content='lite_embeddings', content_rowid='id',
tokenize='unicode61'
)`
if err := db.Exec(sql).Error; err != nil {
logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
}
}
func vecTableName(dim int) string {
return fmt.Sprintf("vec_embeddings_%d", dim)
}
func (r *sqliteRepository) ensureVecTable(dim int) {
if dim <= 0 || r.vecTables[dim] {
return
}
tbl := vecTableName(dim)
createSQL := fmt.Sprintf(
`CREATE VIRTUAL TABLE IF NOT EXISTS %s USING vec0(embedding float[%d] distance_metric=cosine)`,
tbl, dim,
)
if err := r.db.Exec(createSQL).Error; err != nil {
if strings.Contains(err.Error(), "already exists") {
r.vecTables[dim] = true
return
}
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
return
}
r.vecTables[dim] = true
}
func (r *sqliteRepository) ensureExistingVecTables() {
var dims []int
r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
for _, dim := range dims {
r.ensureVecTable(dim)
}
}
func (r *sqliteRepository) EngineType() types.RetrieverEngineType {
return types.SQLiteRetrieverEngineType
}
func (r *sqliteRepository) Support() []types.RetrieverType {
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
}
func (r *sqliteRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
row := toSQLiteEmbedding(indexInfo)
emb := extractEmbedding(params, indexInfo.SourceID)
if len(emb) > 0 {
row.Dimension = len(emb)
}
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(row).Error; err != nil {
return err
}
r.syncFTS5Insert(ctx, row)
if len(emb) > 0 && row.ID > 0 {
r.insertVec(ctx, row.ID, row.Dimension, emb)
}
return nil
}
func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error {
if len(indexInfoList) == 0 {
return nil
}
rows := make([]*sqliteEmbedding, len(indexInfoList))
embs := make([][]float32, len(indexInfoList))
for i, info := range indexInfoList {
rows[i] = toSQLiteEmbedding(info)
emb := extractEmbedding(params, info.SourceID)
embs[i] = emb
if len(emb) > 0 {
rows[i].Dimension = len(emb)
}
}
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error; err != nil {
return err
}
for i, row := range rows {
r.syncFTS5Insert(ctx, row)
if len(embs[i]) > 0 && row.ID > 0 {
r.insertVec(ctx, row.ID, row.Dimension, embs[i])
}
}
return nil
}
func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList []*types.IndexInfo, _ map[string]any) int64 {
var total int64
for _, info := range indexInfoList {
total += int64(len(info.Content)) + 200
}
return total
}
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
r.deleteRowsAndVecs(ctx, rows)
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) CopyIndices(ctx context.Context,
_ string,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
_ int, _ string,
) error {
for sourceChunkID, targetChunkID := range sourceToTargetChunkIDMap {
var src sqliteEmbedding
if err := r.db.WithContext(ctx).Where("chunk_id = ?", sourceChunkID).First(&src).Error; err != nil {
continue
}
newRow := sqliteEmbedding{
SourceID: uuid.New().String(),
SourceType: src.SourceType,
ChunkID: targetChunkID,
KnowledgeID: sourceToTargetKBIDMap[src.KnowledgeID],
KnowledgeBaseID: targetKnowledgeBaseID,
TagID: src.TagID,
Content: src.Content,
Dimension: src.Dimension,
IsEnabled: src.IsEnabled,
}
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy chunk %s: %v", sourceChunkID, err)
continue
}
r.syncFTS5Insert(ctx, &newRow)
if src.Dimension > 0 && newRow.ID > 0 {
r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
}
}
return nil
}
func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
for chunkID, enabled := range chunkStatusMap {
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)
}
return nil
}
func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
for chunkID, tagID := range chunkTagMap {
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)
}
return nil
}
// --- Retrieve ---
func (r *sqliteRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
var results []*types.RetrieveResult
if params.RetrieverType == types.KeywordsRetrieverType || params.RetrieverType == "" {
res, err := r.keywordsRetrieve(ctx, params)
if err != nil {
results = append(results, &types.RetrieveResult{
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
Error: err,
})
} else {
results = append(results, res...)
}
}
if params.RetrieverType == types.VectorRetrieverType || params.RetrieverType == "" {
res, err := r.vectorRetrieve(ctx, params)
if err != nil {
results = append(results, &types.RetrieveResult{
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
Error: err,
})
} else {
results = append(results, res...)
}
}
return results, nil
}
// --- Keywords retrieval via FTS5 ---
func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
if params.Query == "" {
return nil, nil
}
ftsQuery := sanitizeFTS5Query(params.Query)
sql := `
SELECT e.id, e.source_id, e.source_type, e.chunk_id,
e.knowledge_id, e.knowledge_base_id, e.tag_id,
e.content,
bm25(lite_embeddings_fts) AS score
FROM lite_embeddings_fts
JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid
WHERE lite_embeddings_fts MATCH ?
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
`
args := []interface{}{ftsQuery}
for _, wp := range buildFilterWhere(params) {
sql += " AND " + wp.clause
args = append(args, wp.args...)
}
sql += " ORDER BY score ASC LIMIT ?"
args = append(args, params.TopK)
type ftsResult struct {
ID uint
SourceID string
SourceType int
ChunkID string
KnowledgeID string
KnowledgeBaseID string
TagID string
Content string
Score float64
}
var rows []ftsResult
if err := r.db.WithContext(ctx).Raw(sql, args...).Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("FTS5 query failed: %w", err)
}
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: query=%q, ftsQuery=%q, matched=%d rows", params.Query, ftsQuery, len(rows))
items := make([]*types.IndexWithScore, len(rows))
for i, row := range rows {
// bm25 越小越相关 → 转成正向分数
score := -row.Score
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s",
i+1, row.ChunkID, row.Score, score, row.Content)
items[i] = &types.IndexWithScore{
ID: fmt.Sprintf("%d", row.ID),
SourceID: row.SourceID,
SourceType: types.SourceType(row.SourceType),
ChunkID: row.ChunkID,
KnowledgeID: row.KnowledgeID,
KnowledgeBaseID: row.KnowledgeBaseID,
TagID: row.TagID,
Content: row.Content,
Score: score,
MatchType: types.MatchTypeKeywords,
}
}
return []*types.RetrieveResult{{
Results: items,
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.KeywordsRetrieverType,
}}, nil
}
func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
if len(params.Embedding) == 0 {
return nil, nil
}
dim := len(params.Embedding)
r.ensureVecTable(dim)
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
if err != nil {
return nil, fmt.Errorf("serialize query vector failed: %w", err)
}
tbl := vecTableName(dim)
// ⚠️ sqlite-vec 要求必须有 k = ?
vecSQL := fmt.Sprintf(`
SELECT v.rowid, v.distance,
e.source_id, e.source_type, e.chunk_id,
e.knowledge_id, e.knowledge_base_id,
e.tag_id, e.content
FROM %s v
JOIN lite_embeddings e ON e.id = v.rowid
WHERE v.embedding MATCH ?
AND k = ?
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
`, tbl)
args := []interface{}{
queryBlob,
params.TopK, // 这里就是 k
}
// 追加过滤条件
for _, wp := range buildFilterWhere(params) {
vecSQL += " AND " + wp.clause
args = append(args, wp.args...)
}
// ⚠️ 这里仍然建议加 ORDER BY,虽然 vec0 已经按距离返回
vecSQL += " ORDER BY v.distance ASC"
type row struct {
Rowid uint
Distance float64
SourceID string
SourceType int
ChunkID string
KnowledgeID string
KnowledgeBaseID string
TagID string
Content string
}
var rows []row
if err := r.db.WithContext(ctx).
Raw(vecSQL, args...).
Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("sqlite-vec query failed: %w", err)
}
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: query_dim=%d, threshold=%.4f, matched=%d rows", dim, params.Threshold, len(rows))
items := make([]*types.IndexWithScore, 0, len(rows))
for i, v := range rows {
// cosine distance = 1 - cosine_similarity
score := 1 - v.Distance
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: #%d chunk_id=%s, distance=%.4f, score=%.4f, content_preview=%.60s",
i+1, v.ChunkID, v.Distance, score, v.Content)
items = append(items, &types.IndexWithScore{
ID: fmt.Sprintf("%d", v.Rowid),
SourceID: v.SourceID,
SourceType: types.SourceType(v.SourceType),
ChunkID: v.ChunkID,
KnowledgeID: v.KnowledgeID,
KnowledgeBaseID: v.KnowledgeBaseID,
TagID: v.TagID,
Content: v.Content,
Score: score,
MatchType: types.MatchTypeEmbedding,
})
}
return []*types.RetrieveResult{{
Results: items,
RetrieverEngineType: types.SQLiteRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
}}, nil
}
// --- Internal helpers ---
func toSQLiteEmbedding(info *types.IndexInfo) *sqliteEmbedding {
enabled := true
return &sqliteEmbedding{
SourceID: info.SourceID,
SourceType: int(info.SourceType),
ChunkID: info.ChunkID,
KnowledgeID: info.KnowledgeID,
KnowledgeBaseID: info.KnowledgeBaseID,
TagID: info.TagID,
Content: common.CleanInvalidUTF8(info.Content),
Dimension: 0,
IsEnabled: &enabled,
}
}
func extractEmbedding(params map[string]any, sourceID string) []float32 {
if params == nil {
return nil
}
embMap, ok := params["embedding"].(map[string][]float32)
if !ok {
return nil
}
return embMap[sourceID]
}
func (r *sqliteRepository) insertVec(_ context.Context, rowID uint, dim int, emb []float32) {
r.ensureVecTable(dim)
blob, err := sqlite_vec.SerializeFloat32(emb)
if err != nil {
return
}
sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
r.db.Exec(sql, rowID, blob)
}
func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmbedding) {
dimIDs := make(map[int][]uint)
for _, row := range rows {
if row.Dimension > 0 {
dimIDs[row.Dimension] = append(dimIDs[row.Dimension], row.ID)
}
}
for dim, ids := range dimIDs {
if !r.vecTables[dim] {
continue
}
tbl := vecTableName(dim)
for _, id := range ids {
r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
}
}
}
func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
if !r.vecTables[dim] {
return
}
tbl := vecTableName(dim)
r.db.Exec(fmt.Sprintf(
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
tbl, tbl,
), dstID, srcID)
}
func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbedding) {
if row.ID == 0 {
return
}
sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
}
type whereClause struct {
clause string
args []interface{}
}
func buildFilterWhere(params types.RetrieveParams) []whereClause {
var parts []whereClause
if len(params.KnowledgeBaseIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.knowledge_base_id IN (" + placeholders(len(params.KnowledgeBaseIDs)) + ")",
args: toInterfaceSlice(params.KnowledgeBaseIDs),
})
}
if len(params.KnowledgeIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.knowledge_id IN (" + placeholders(len(params.KnowledgeIDs)) + ")",
args: toInterfaceSlice(params.KnowledgeIDs),
})
}
if len(params.TagIDs) > 0 {
parts = append(parts, whereClause{
clause: "e.tag_id IN (" + placeholders(len(params.TagIDs)) + ")",
args: toInterfaceSlice(params.TagIDs),
})
}
return parts
}
func placeholders(n int) string {
p := make([]string, n)
for i := range p {
p[i] = "?"
}
return strings.Join(p, ",")
}
func toInterfaceSlice(ss []string) []interface{} {
out := make([]interface{}, len(ss))
for i, s := range ss {
out[i] = s
}
return out
}
// sanitizeFTS5Query builds an FTS5 query from user input.
// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used
// by Elasticsearch etc.) and joined with OR for broad partial matching.
// Non-CJK words are kept intact. BM25 ranking naturally boosts documents
// that match more tokens.
func sanitizeFTS5Query(q string) string {
q = strings.TrimSpace(q)
if q == "" {
return q
}
var cjkRunes []rune
var nonCJKWords []string
var buf strings.Builder
flushNonCJK := func() {
if buf.Len() > 0 {
nonCJKWords = append(nonCJKWords, buf.String())
buf.Reset()
}
}
for _, r := range q {
if unicode.Is(unicode.Han, r) {
flushNonCJK()
cjkRunes = append(cjkRunes, r)
} else if unicode.IsSpace(r) || r == '|' {
flushNonCJK()
} else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' {
// skip FTS5 special characters
} else {
buf.WriteRune(r)
}
}
flushNonCJK()
var parts []string
// CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...)
if len(cjkRunes) == 1 {
parts = append(parts, `"`+string(cjkRunes[0])+`"`)
} else {
for i := 0; i < len(cjkRunes)-1; i++ {
bigram := string(cjkRunes[i]) + string(cjkRunes[i+1])
parts = append(parts, `"`+bigram+`"`)
}
}
for _, w := range nonCJKWords {
w = strings.ReplaceAll(w, `"`, `""`)
parts = append(parts, `"`+w+`"`)
}
if len(parts) == 0 {
return `"` + strings.ReplaceAll(q, `"`, `""`) + `"`
}
return strings.Join(parts, " OR ")
}

View File

@ -114,7 +114,7 @@ func (r *tenantRepository) AdjustStorageUsed(ctx context.Context, tenantID uint6
tenant.StorageUsed += delta
// 保存更新并验证业务规则
if tenant.StorageUsed < 0 {
logger.Error(ctx, "tenant storage used is negative %s: %d", tenant.ID, tenant.StorageUsed)
logger.Errorf(ctx, "tenant storage used is negative %d: %d", tenant.ID, tenant.StorageUsed)
tenant.StorageUsed = 0
}

View File

@ -183,6 +183,9 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
return
}
if len(res) > 0 {
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"query": q,
@ -338,6 +341,9 @@ func (p *PluginSearch) searchByTargets(
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
if len(directResults) > 0 {
for _, r := range directResults {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"loaded_count": len(directResults),
@ -383,6 +389,9 @@ func (p *PluginSearch) searchByTargets(
})
return
}
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,

View File

@ -222,5 +222,6 @@ func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.S
KnowledgeFilename: knowledge.FileName,
KnowledgeSource: knowledge.Source,
ChunkMetadata: chunk.Metadata,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
}
}

View File

@ -79,7 +79,7 @@ const (
// NewChunkExtractTask creates a new chunk extract task
func NewChunkExtractTask(
ctx context.Context,
client *asynq.Client,
client interfaces.TaskEnqueuer,
tenantID uint64,
chunkID string,
modelID string,
@ -109,7 +109,7 @@ func NewChunkExtractTask(
// NewTableExtractTask creates a new table extract task
func NewDataTableSummaryTask(
ctx context.Context,
client *asynq.Client,
client interfaces.TaskEnqueuer,
tenantID uint64,
knowledgeID string,
summaryModel string,

View File

@ -12,6 +12,7 @@ import (
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
@ -23,11 +24,28 @@ type cosFileService struct {
client *cos.Client
bucketURL string
cosPathPrefix string
// 临时桶配置(用于存放自动过期的临时文件)
tempClient *cos.Client
tempBucketURL string
}
// newCosClient creates a bare cosFileService with just the SDK client initialised.
// Shared by NewCosFileService* constructors and CheckCosConnectivity.
func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
u, err := url.Parse(bucketURL)
logger.Infof(context.Background(), "newCosClient: bucketURL: %s", bucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
}
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretID,
SecretKey: secretKey,
},
})
return &cosFileService{client: client, bucketURL: bucketURL}, nil
}
// NewCosFileService creates a new COS file service instance
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
@ -35,49 +53,51 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
u, err := url.Parse(bucketURL)
svc, err := newCosClient(bucketName, region, secretId, secretKey)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
return nil, err
}
b := &cos.BaseURL{BucketURL: u}
client := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretId,
SecretKey: secretKey,
},
})
svc.cosPathPrefix = cosPathPrefix
svc := &cosFileService{
client: client,
bucketURL: bucketURL,
cosPathPrefix: cosPathPrefix,
}
// 如果配置了临时桶,初始化临时桶客户端
if tempBucketName != "" {
if tempRegion == "" {
tempRegion = region // 默认使用相同的 region
tempRegion = region
}
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", tempBucketName, tempRegion)
tempU, err := url.Parse(tempBucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
}
tempB := &cos.BaseURL{BucketURL: tempU}
tempClient := cos.NewClient(tempB, &http.Client{
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretId,
SecretKey: secretKey,
},
})
svc.tempClient = tempClient
svc.tempBucketURL = tempBucketURL
}
return svc, nil
}
// CheckConnectivity verifies COS is reachable by performing a HEAD request on the bucket.
func (s *cosFileService) CheckConnectivity(ctx context.Context) error {
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err := s.client.Bucket.Head(checkCtx)
return err
}
// CheckCosConnectivity tests COS connectivity using the provided credentials.
// It creates a temporary service instance internally and delegates to CheckConnectivity.
func CheckCosConnectivity(ctx context.Context, bucketName, region, secretID, secretKey string) error {
svc, err := newCosClient(bucketName, region, secretID, secretKey)
if err != nil {
return err
}
return svc.CheckConnectivity(ctx)
}
// SaveFile saves a file to COS storage
// It generates a unique name for the file and organizes it by tenant and knowledge ID
func (s *cosFileService) SaveFile(ctx context.Context,

View File

@ -14,6 +14,11 @@ import (
// used for testing or when file storage is not required
type DummyFileService struct{}
// CheckConnectivity always succeeds for the dummy service.
func (s *DummyFileService) CheckConnectivity(ctx context.Context) error {
return nil
}
// NewDummyFileService creates a new instance of DummyFileService
func NewDummyFileService() interfaces.FileService {
return &DummyFileService{}

View File

@ -19,6 +19,18 @@ type localFileService struct {
baseDir string // Base directory for file storage
}
// CheckConnectivity verifies the local storage directory exists and is accessible.
func (s *localFileService) CheckConnectivity(ctx context.Context) error {
info, err := os.Stat(s.baseDir)
if err != nil {
return fmt.Errorf("storage directory not accessible: %w", err)
}
if !info.IsDir() {
return fmt.Errorf("storage path is not a directory: %s", s.baseDir)
}
return nil
}
// NewLocalFileService creates a new local file service instance
func NewLocalFileService(baseDir string) interfaces.FileService {
return &localFileService{

View File

@ -22,11 +22,10 @@ type minioFileService struct {
bucketName string
}
// NewMinioFileService creates a MinIO file service
func NewMinioFileService(endpoint,
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
) (interfaces.FileService, error) {
// Initialize MinIO client
// newMinioClient creates a bare minioFileService with just the SDK client initialised.
// Shared by NewMinioFileService (which also ensures the bucket exists) and
// CheckMinioConnectivity (read-only probe).
func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) {
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
@ -34,24 +33,60 @@ func NewMinioFileService(endpoint,
if err != nil {
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
}
return &minioFileService{client: client, bucketName: bucketName}, nil
}
// Check if bucket exists, create if not
exists, err := client.BucketExists(context.Background(), bucketName)
// NewMinioFileService creates a MinIO file service.
// It verifies that the bucket exists and creates it if missing.
func NewMinioFileService(endpoint,
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
) (interfaces.FileService, error) {
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return nil, err
}
exists, err := svc.client.BucketExists(context.Background(), bucketName)
if err != nil {
return nil, fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
err = client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{})
if err != nil {
if err = svc.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}); err != nil {
return nil, fmt.Errorf("failed to create bucket: %w", err)
}
}
return &minioFileService{
client: client,
bucketName: bucketName,
}, nil
return svc, nil
}
// CheckConnectivity verifies MinIO is reachable and, if a bucket is configured,
// that the bucket exists. This is a read-only probe — it never creates a bucket.
func (s *minioFileService) CheckConnectivity(ctx context.Context) error {
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if s.bucketName != "" {
exists, err := s.client.BucketExists(checkCtx, s.bucketName)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("bucket %q does not exist", s.bucketName)
}
return nil
}
_, err := s.client.ListBuckets(checkCtx)
return err
}
// CheckMinioConnectivity tests MinIO connectivity using the provided credentials.
// It creates a temporary service instance internally and delegates to CheckConnectivity.
func CheckMinioConnectivity(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) error {
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return err
}
return svc.CheckConnectivity(ctx)
}
// SaveFile saves a file to MinIO

View File

@ -0,0 +1,365 @@
package service
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
const (
vlmOCRPrompt = "请提取这张文档图片中的所有正文内容,用纯 Markdown 格式输出。要求:\n" +
"1. 忽略页眉、页脚\n" +
"2. 表格使用 Markdown 表格语法\n" +
"3. 公式使用 LaTeX 格式(用 $ 或 $$ 包裹)\n" +
"4. 按照原文阅读顺序组织\n" +
"5. 只输出提取到的文本内容,不要添加任何 HTML 标签\n" +
"如果图片中没有可识别的文字内容,请回复:无文字内容。"
vlmCaptionPrompt = "简单凝炼的描述图片的主要内容"
)
// ImageMultimodalService handles image:multimodal asynq tasks.
// It reads images from local storage, performs OCR and VLM caption,
// and creates child chunks for the processed content.
type ImageMultimodalService struct {
chunkService interfaces.ChunkService
modelService interfaces.ModelService
kbService interfaces.KnowledgeBaseService
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ollamaService *ollama.OllamaService
}
func NewImageMultimodalService(
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
kbService interfaces.KnowledgeBaseService,
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ollamaService *ollama.OllamaService,
) interfaces.TaskHandler {
return &ImageMultimodalService{
chunkService: chunkService,
modelService: modelService,
kbService: kbService,
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ollamaService: ollamaService,
}
}
// Handle implements asynq handler for TypeImageMultimodal.
func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
var payload types.ImageMultimodalPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return fmt.Errorf("unmarshal image multimodal payload: %w", err)
}
logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, local_path=%s, ocr=%v, caption=%v",
payload.ChunkID, payload.ImageURL, payload.ImageLocalPath, payload.EnableOCR, payload.EnableCaption)
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
if err != nil {
return fmt.Errorf("resolve VLM: %w", err)
}
imgBytes, err := os.ReadFile(payload.ImageLocalPath)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), trying to download from URL", payload.ImageLocalPath, err)
imgBytes, err = downloadImageFromURL(payload.ImageURL)
if err != nil {
logger.Errorf(ctx, "[ImageMultimodal] Failed to download image from URL %s: %v", payload.ImageURL, err)
return fmt.Errorf("read image: local file %s not found and URL download failed: %w", payload.ImageLocalPath, err)
}
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
} else {
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
defer os.Remove(payload.ImageLocalPath)
}
imageInfo := types.ImageInfo{
URL: payload.ImageURL,
OriginalURL: payload.ImageURL,
}
if payload.EnableOCR {
ocrText, ocrErr := vlmModel.Predict(ctx, imgBytes, vlmOCRPrompt)
if ocrErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
} else {
ocrText = sanitizeOCRText(ocrText)
if ocrText != "" {
imageInfo.OCRText = ocrText
} else {
logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
}
}
}
if payload.EnableCaption {
caption, capErr := vlmModel.Predict(ctx, imgBytes, vlmCaptionPrompt)
if capErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
} else if caption != "" {
imageInfo.Caption = caption
}
}
// Build child chunks for OCR and caption results
imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo})
var newChunks []*types.Chunk
if imageInfo.OCRText != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: imageInfo.OCRText,
ChunkType: types.ChunkTypeImageOCR,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if imageInfo.Caption != "" {
newChunks = append(newChunks, &types.Chunk{
ID: uuid.New().String(),
TenantID: payload.TenantID,
KnowledgeID: payload.KnowledgeID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Content: imageInfo.Caption,
ChunkType: types.ChunkTypeImageCaption,
ParentChunkID: payload.ChunkID,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
}
if len(newChunks) == 0 {
// Even if OCR/caption both failed, mark knowledge as completed
s.finalizeImageKnowledge(ctx, payload, "")
return nil
}
// Persist chunks
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
return fmt.Errorf("create multimodal chunks: %w", err)
}
for _, c := range newChunks {
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
c.ChunkType, c.ID, payload.ImageURL, len(c.Content))
}
// Index chunks so they can be retrieved
s.indexChunks(ctx, payload, newChunks)
// Update the parent text chunk's ImageInfo (mirrors old docreader behaviour)
s.updateParentChunkImageInfo(ctx, payload, imageInfo)
// For standalone image files, use caption as the knowledge description
// and mark the knowledge as completed (it was kept in "processing" until now).
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
return nil
}
// finalizeImageKnowledge updates the knowledge after multimodal processing:
// - For standalone image files: sets Description from caption and marks ParseStatus as completed
// (processChunks kept it in "processing" to wait for multimodal results).
// - For images extracted from PDFs: no-op (description comes from summary generation).
func (s *ImageMultimodalService) finalizeImageKnowledge(ctx context.Context, payload types.ImageMultimodalPayload, caption string) {
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
return
}
if knowledge == nil {
return
}
if !IsImageType(knowledge.FileType) {
return
}
if caption != "" {
knowledge.Description = caption
}
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to finalize knowledge: %v", err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Finalized image knowledge %s (status=completed, description=%d chars)",
payload.KnowledgeID, len(knowledge.Description))
}
}
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
// so they can participate in semantic search.
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if err != nil || kb == nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err)
return
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err)
return
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
return
}
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
return
}
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
for _, chunk := range chunks {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err)
return
}
// Mark chunks as indexed.
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
for _, chunk := range chunks {
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
continue
}
dbChunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
}
}
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
}
// updateParentChunkImageInfo updates the parent text chunk's ImageInfo field,
// replicating the behaviour of the old docreader flow where the parent chunk
// carried the full image metadata (URL, OCR, caption).
func (s *ImageMultimodalService) updateParentChunkImageInfo(ctx context.Context, payload types.ImageMultimodalPayload, imageInfo types.ImageInfo) {
if payload.ChunkID == "" {
return
}
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
return
}
var existingInfos []types.ImageInfo
if chunk.ImageInfo != "" {
_ = json.Unmarshal([]byte(chunk.ImageInfo), &existingInfos)
}
found := false
for i, info := range existingInfos {
if info.URL == imageInfo.URL {
existingInfos[i] = imageInfo
found = true
break
}
}
if !found {
existingInfos = append(existingInfos, imageInfo)
}
imageInfoJSON, _ := json.Marshal(existingInfos)
chunk.ImageInfo = string(imageInfoJSON)
chunk.UpdatedAt = time.Now()
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to update parent chunk %s ImageInfo: %v", chunk.ID, err)
} else {
logger.Infof(ctx, "[ImageMultimodal] Updated parent chunk %s ImageInfo for image %s", chunk.ID, payload.ImageURL)
}
}
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
}
if kb == nil {
return nil, fmt.Errorf("knowledge base %s not found", kbID)
}
vlmCfg := kb.VLMConfig
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
}
// New-style: resolve model through ModelService
if vlmCfg.ModelID != "" {
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
}
// Legacy: create VLM from inline config
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
}
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
func downloadImageFromURL(imageURL string) ([]byte, error) {
if !strings.HasPrefix(imageURL, "http://") && !strings.HasPrefix(imageURL, "https://") {
return nil, fmt.Errorf("unsupported URL scheme: %s", imageURL)
}
client := &http.Client{Timeout: 60 * time.Second}
resp, err := client.Get(imageURL)
if err != nil {
return nil, fmt.Errorf("HTTP GET: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, imageURL)
}
return io.ReadAll(resp.Body)
}

File diff suppressed because it is too large Load Diff

View File

@ -32,7 +32,7 @@ type knowledgeBaseService struct {
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
graphEngine interfaces.RetrieveGraphRepository
asynqClient *asynq.Client
asynqClient interfaces.TaskEnqueuer
}
// NewKnowledgeBaseService creates a new knowledge base service
@ -46,7 +46,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
tenantRepo interfaces.TenantRepository,
fileSvc interfaces.FileService,
graphEngine interfaces.RetrieveGraphRepository,
asynqClient *asynq.Client,
asynqClient interfaces.TaskEnqueuer,
) interfaces.KnowledgeBaseService {
return &knowledgeBaseService{
repo: repo,

View File

@ -9,6 +9,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@ -405,5 +406,56 @@ func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.C
return chatModel, nil
}
// GetVLMModel retrieves and initializes a vision language model instance.
func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error) {
if modelId == "" {
return nil, errors.New("model ID cannot be empty")
}
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
model, err := s.repo.GetByID(ctx, tenantID, modelId)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"model_id": modelId,
"tenant_id": tenantID,
})
return nil, err
}
if model == nil {
return nil, ErrModelNotFound
}
logger.Infof(ctx, "Getting VLM model: %s, source: %s", model.Name, model.Source)
ifType := model.Parameters.InterfaceType
if ifType == "" {
if model.Source == types.ModelSourceLocal {
ifType = "ollama"
} else {
ifType = "openai"
}
}
vlmModel, err := vlm.NewVLM(&vlm.Config{
ModelID: model.ID,
APIKey: model.Parameters.APIKey,
BaseURL: model.Parameters.BaseURL,
ModelName: model.Name,
Source: model.Source,
InterfaceType: ifType,
}, s.ollamaService)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"model_id": model.ID,
"model_name": model.Name,
})
return nil, err
}
return vlmModel, nil
}
// Note: default model selection logic has been removed; models no longer
// maintain a per-type default flag at the service layer.

View File

@ -0,0 +1,105 @@
package service
import (
"regexp"
"strings"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
)
var (
htmlTagPattern = regexp.MustCompile(`<[^>]+>`)
codeBlockPattern = regexp.MustCompile("(?s)^\\s*```[a-zA-Z]*\\s*\n(.*?)\n\\s*```\\s*$")
htmlDocPattern = regexp.MustCompile(`(?i)^\s*(<\!DOCTYPE|<html|<body|<div|<p[\s>]|<table|<h[1-6][\s>])`)
multipleNewlines = regexp.MustCompile(`\n{3,}`)
knownEmptyReplies = []string{
"无文字内容",
"无法识别",
"no text",
"no content",
"empty",
"图片中没有文字",
"图片中没有可识别的文字",
}
)
// sanitizeOCRText cleans up VLM OCR output by stripping HTML wrappers,
// converting HTML to markdown, and filtering out useless responses.
func sanitizeOCRText(raw string) string {
text := strings.TrimSpace(raw)
if text == "" {
return ""
}
text = stripMarkdownCodeBlock(text)
// If stripping HTML tags leaves almost no text, the response is useless
// (e.g. "<html><body><div class="image"><img/></div></body></html>").
plainText := strings.TrimSpace(htmlTagPattern.ReplaceAllString(text, ""))
if len(plainText) < 10 && htmlTagPattern.MatchString(text) {
return ""
}
if looksLikeHTML(text) {
text = ocrHTMLToMarkdown(text)
text = strings.TrimSpace(text)
if text == "" {
return ""
}
}
if isKnownEmptyReply(text) {
return ""
}
text = multipleNewlines.ReplaceAllString(text, "\n\n")
return strings.TrimSpace(text)
}
// stripMarkdownCodeBlock removes a markdown code-fence wrapper that some
// models add around their output (e.g. ```html\n...\n``` or ```markdown\n...\n```).
func stripMarkdownCodeBlock(text string) string {
if m := codeBlockPattern.FindStringSubmatch(text); len(m) == 2 {
return strings.TrimSpace(m[1])
}
return text
}
// looksLikeHTML returns true when the text appears to be an HTML document
// or contains a significant amount of HTML tags.
func looksLikeHTML(text string) bool {
if htmlDocPattern.MatchString(text) {
return true
}
tags := htmlTagPattern.FindAllString(text, -1)
if len(tags) == 0 {
return false
}
tagChars := 0
for _, t := range tags {
tagChars += len(t)
}
return float64(tagChars)/float64(len(text)) > 0.3
}
// ocrHTMLToMarkdown converts HTML content to markdown, falling back to the
// original text on failure.
func ocrHTMLToMarkdown(content string) string {
md, err := htmltomd.ConvertString(content)
if err != nil {
return content
}
return md
}
// isKnownEmptyReply checks whether the text matches a known "no content"
// reply pattern that VLM models produce when the image has no text.
func isKnownEmptyReply(text string) bool {
lower := strings.ToLower(strings.TrimSpace(text))
for _, phrase := range knownEmptyReplies {
if lower == strings.ToLower(phrase) {
return true
}
}
return false
}

View File

@ -0,0 +1,213 @@
package service
import "testing"
func TestSanitizeOCRText(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "empty string",
input: "",
want: "",
},
{
name: "whitespace only",
input: " \n\t ",
want: "",
},
{
name: "pure HTML skeleton with no text",
input: `<html><body><div class="image"><img/></div></body></html>`,
want: "",
},
{
name: "HTML with only whitespace text",
input: "<html><body> \n </body></html>",
want: "",
},
{
name: "valid markdown passes through",
input: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
want: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
},
{
name: "code block wrapper stripped",
input: "```markdown\n# 文档标题\n\n正文内容在这里。\n```",
want: "# 文档标题\n\n正文内容在这里。",
},
{
name: "html code block wrapper stripped",
input: "```html\n<p>这是一段内容</p>\n```",
want: "这是一段内容",
},
{
name: "HTML document converted to markdown",
input: "<html><body><h1>标题</h1><p>这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。</p></body></html>",
want: "# 标题\n\n这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。",
},
{
name: "known empty reply - Chinese",
input: "无文字内容",
want: "",
},
{
name: "known empty reply - no text",
input: "No text",
want: "",
},
{
name: "known empty reply - 图片中没有文字",
input: "图片中没有文字",
want: "",
},
{
name: "plain text with minimal HTML not converted",
input: "这是一段正常文本,价格 <100 元。",
want: "这是一段正常文本,价格 <100 元。",
},
{
name: "multiple blank lines collapsed",
input: "段落一\n\n\n\n\n段落二",
want: "段落一\n\n段落二",
},
{
name: "HTML with substantial text content is converted",
input: "<div><h2>报告摘要</h2><p>本季度营收同比增长 15%,净利润达到 2.3 亿元。</p><table><tr><th>指标</th><th>数值</th></tr><tr><td>营收</td><td>10亿</td></tr></table></div>",
want: "", // placeholder; will be checked for non-empty
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := sanitizeOCRText(tt.input)
if tt.name == "HTML with substantial text content is converted" {
if got == "" {
t.Errorf("sanitizeOCRText() returned empty for substantial HTML content")
}
if got == tt.input {
t.Errorf("sanitizeOCRText() did not convert HTML, got original")
}
return
}
if got != tt.want {
t.Errorf("sanitizeOCRText() = %q, want %q", got, tt.want)
}
})
}
}
func TestStripMarkdownCodeBlock(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "no code block",
input: "just normal text",
want: "just normal text",
},
{
name: "markdown code block",
input: "```markdown\n# Title\nContent here\n```",
want: "# Title\nContent here",
},
{
name: "html code block",
input: "```html\n<p>hello</p>\n```",
want: "<p>hello</p>",
},
{
name: "plain code block",
input: "```\nsome text\n```",
want: "some text",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := stripMarkdownCodeBlock(tt.input)
if got != tt.want {
t.Errorf("stripMarkdownCodeBlock() = %q, want %q", got, tt.want)
}
})
}
}
func TestLooksLikeHTML(t *testing.T) {
tests := []struct {
name string
input string
want bool
}{
{
name: "HTML document",
input: "<html><body><p>text</p></body></html>",
want: true,
},
{
name: "DOCTYPE",
input: "<!DOCTYPE html><html><body></body></html>",
want: true,
},
{
name: "body tag",
input: "<body><p>content</p></body>",
want: true,
},
{
name: "plain markdown",
input: "# Title\n\nSome paragraph text",
want: false,
},
{
name: "text with minor HTML",
input: "This is mostly text with a <b>bold</b> word.",
want: false,
},
{
name: "heavy HTML tags",
input: "<div><p><span>x</span></p></div><div><p><span>y</span></p></div>",
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := looksLikeHTML(tt.input)
if got != tt.want {
t.Errorf("looksLikeHTML() = %v, want %v", got, tt.want)
}
})
}
}
func TestIsKnownEmptyReply(t *testing.T) {
tests := []struct {
input string
want bool
}{
{"无文字内容", true},
{"无法识别", true},
{"no text", true},
{"No Text", true},
{"NO CONTENT", true},
{"empty", true},
{"这是正常内容", false},
{"", false},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
got := isKnownEmptyReply(tt.input)
if got != tt.want {
t.Errorf("isKnownEmptyReply(%q) = %v, want %v", tt.input, got, tt.want)
}
})
}
}

View File

@ -25,7 +25,7 @@ type knowledgeTagService struct {
chunkRepo interfaces.ChunkRepository
retrieveEngine interfaces.RetrieveEngineRegistry
modelService interfaces.ModelService
task *asynq.Client
task interfaces.TaskEnqueuer
kbShareService interfaces.KBShareService
}
@ -37,7 +37,7 @@ func NewKnowledgeTagService(
chunkRepo interfaces.ChunkRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
modelService interfaces.ModelService,
task *asynq.Client,
task interfaces.TaskEnqueuer,
kbShareService interfaces.KBShareService,
) (interfaces.KnowledgeTagService, error) {
return &knowledgeTagService{

View File

@ -29,8 +29,12 @@ type Config struct {
PromptTemplates *PromptTemplatesConfig `yaml:"prompt_templates" json:"prompt_templates"`
}
// DocReaderConfig configures the document parser client (gRPC or HTTP).
type DocReaderConfig struct {
// Addr: for gRPC it is the server address (e.g. "localhost:50051"); for HTTP it is the base URL (e.g. "http://localhost:8080").
Addr string `yaml:"addr" json:"addr"`
// Transport: "grpc" (default) or "http"
Transport string `yaml:"transport" json:"transport"`
}
type VectorDatabaseConfig struct {

View File

@ -9,11 +9,13 @@ import (
"fmt"
"net/url"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
_ "github.com/duckdb/duckdb-go/v2"
esv7 "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v8"
@ -25,9 +27,9 @@ import (
"go.uber.org/dig"
"google.golang.org/grpc"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"github.com/Tencent/WeKnora/docreader/client"
"github.com/Tencent/WeKnora/internal/application/repository"
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
@ -36,6 +38,7 @@ import (
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
sqliteRetrieverRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/sqlite"
"github.com/Tencent/WeKnora/internal/application/service"
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
"github.com/Tencent/WeKnora/internal/application/service/file"
@ -48,6 +51,7 @@ import (
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/handler"
"github.com/Tencent/WeKnora/internal/handler/session"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/mcp"
"github.com/Tencent/WeKnora/internal/models/embedding"
@ -94,6 +98,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// External service clients
logger.Debugf(ctx, "[Container] Registering external service clients...")
must(container.Provide(initDocReaderClient))
must(container.Provide(initImageResolver))
must(container.Provide(initOllamaService))
must(container.Provide(initNeo4jClient))
must(container.Provide(stream.NewStreamManager))
@ -146,6 +151,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// Extract services - register individual extracters with names
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
must(container.Provide(service.NewMessageService))
must(container.Provide(service.NewMCPServiceService))
@ -169,9 +175,16 @@ func BuildContainer(container *dig.Container) *dig.Container {
logger.Debugf(ctx, "[Container] Registering session service...")
must(container.Provide(service.NewSessionService))
logger.Debugf(ctx, "[Container] Registering asynq client and server...")
must(container.Provide(router.NewAsyncqClient))
must(container.Provide(router.NewAsynqServer))
logger.Debugf(ctx, "[Container] Registering task enqueuer...")
redisAvailable := os.Getenv("REDIS_ADDR") != ""
if redisAvailable {
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
must(container.Provide(router.NewAsynqServer))
} else {
syncExec := router.NewSyncTaskExecutor()
must(container.Provide(func() interfaces.TaskEnqueuer { return syncExec }))
must(container.Provide(func() *router.SyncTaskExecutor { return syncExec }))
}
// Chat pipeline components for processing chat requests
logger.Debugf(ctx, "[Container] Registering chat pipeline plugins...")
@ -218,9 +231,13 @@ func BuildContainer(container *dig.Container) *dig.Container {
logger.Debugf(ctx, "[Container] HTTP handlers registered")
// Router configuration
logger.Debugf(ctx, "[Container] Registering router and starting asynq server...")
logger.Debugf(ctx, "[Container] Registering router and starting task server...")
must(container.Provide(router.NewRouter))
must(container.Invoke(router.RunAsynqServer))
if redisAvailable {
must(container.Invoke(router.RunAsynqServer))
} else {
must(container.Invoke(router.RegisterSyncHandlers))
}
logger.Infof(ctx, "[Container] Container initialization completed successfully")
return container
@ -249,19 +266,23 @@ func initTracer() (*tracing.Tracer, error) {
}
func initRedisClient() (*redis.Client, error) {
redisAddr := os.Getenv("REDIS_ADDR")
if redisAddr == "" {
logger.Infof(context.Background(), "[Redis] No REDIS_ADDR configured, Redis disabled (Lite mode)")
return nil, nil
}
db, err := strconv.Atoi(os.Getenv("REDIS_DB"))
if err != nil {
return nil, err
db = 0
}
client := redis.NewClient(&redis.Options{
Addr: os.Getenv("REDIS_ADDR"),
Addr: redisAddr,
Username: os.Getenv("REDIS_USERNAME"),
Password: os.Getenv("REDIS_PASSWORD"),
DB: db,
})
// 验证连接
_, err = client.Ping(context.Background()).Result()
if err != nil {
return nil, fmt.Errorf("连接Redis失败: %w", err)
@ -271,6 +292,10 @@ func initRedisClient() (*redis.Client, error) {
}
func initContextStorage(redisClient *redis.Client) (llmcontext.ContextStorage, error) {
if redisClient == nil {
logger.Infof(context.Background(), "[ContextStorage] Redis not available, using in-memory storage")
return llmcontext.NewMemoryStorage(), nil
}
storage, err := llmcontext.NewRedisStorage(redisClient, 24*time.Hour, "context:")
if err != nil {
return nil, err
@ -334,6 +359,21 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
os.Getenv("DB_PORT"),
os.Getenv("DB_NAME"),
)
case "sqlite":
dbPath := os.Getenv("DB_PATH")
if dbPath == "" {
dbPath = "./data/weknora.db"
}
if dir := filepath.Dir(dbPath); dir != "." && dir != "" {
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to create SQLite data directory %s: %w", dir, err)
}
}
sqlite_vec.Auto()
dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on"
dialector = sqlite.Open(dsn)
migrateDSN = "sqlite3://" + dbPath
logger.Infof(context.Background(), "DB Config: driver=sqlite path=%s", dbPath)
default:
return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER"))
}
@ -342,6 +382,16 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
return nil, err
}
if os.Getenv("DB_DRIVER") == "sqlite" {
sqlDB, err := db.DB()
if err != nil {
return nil, fmt.Errorf("failed to get underlying sql.DB: %w", err)
}
if err := sqlDB.Ping(); err != nil {
return nil, fmt.Errorf("failed to ping SQLite database: %w", err)
}
}
// Run database migrations automatically (optional, can be disabled via env var)
// To disable auto-migration, set AUTO_MIGRATE=false
// To enable auto-recovery from dirty state, set AUTO_RECOVER_DIRTY=true
@ -374,7 +424,14 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
}
// Configure connection pool parameters
sqlDB.SetMaxIdleConns(10)
if os.Getenv("DB_DRIVER") == "sqlite" {
// SQLite only supports one concurrent writer even in WAL mode.
// Limiting to a single open connection serialises all DB access and
// prevents "database is locked" errors from concurrent goroutines.
sqlDB.SetMaxOpenConns(1)
} else {
sqlDB.SetMaxIdleConns(10)
}
sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute)
return db, nil
@ -390,7 +447,11 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
// - Configured file service implementation
// - Error if initialization fails
func initFileService(cfg *config.Config) (interfaces.FileService, error) {
switch os.Getenv("STORAGE_TYPE") {
storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE"))
if storageType == "" {
storageType = "local"
}
switch storageType {
case "minio":
if os.Getenv("MINIO_ENDPOINT") == "" ||
os.Getenv("MINIO_ACCESS_KEY_ID") == "" ||
@ -419,8 +480,8 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("COS_SECRET_ID"),
os.Getenv("COS_SECRET_KEY"),
os.Getenv("COS_PATH_PREFIX"),
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
os.Getenv("COS_TEMP_BUCKET_NAME"),
os.Getenv("COS_TEMP_REGION"),
)
case "tos":
if os.Getenv("TOS_ENDPOINT") == "" ||
@ -441,11 +502,15 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
)
case "local":
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
}
return file.NewLocalFileService(baseDir), nil
case "dummy":
return file.NewDummyFileService(), nil
default:
return nil, fmt.Errorf("unsupported storage type: %s", os.Getenv("STORAGE_TYPE"))
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
}
}
@ -474,6 +539,16 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
log.Infof("Register postgres retrieve engine success")
}
}
if slices.Contains(retrieveDriver, "sqlite") {
sqliteRepo := sqliteRetrieverRepo.NewSQLiteRetrieveEngineRepository(db)
if err := registry.Register(
retriever.NewKVHybridRetrieveEngine(sqliteRepo, types.SQLiteRetrieverEngineType),
); err != nil {
log.Errorf("Register sqlite retrieve engine failed: %v", err)
} else {
log.Infof("Register sqlite retrieve engine success")
}
}
if slices.Contains(retrieveDriver, "elasticsearch_v8") {
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")},
@ -639,21 +714,39 @@ func registerPoolCleanup(pool *ants.Pool, cleaner interfaces.ResourceCleaner) {
})
}
// initDocReaderClient initializes the document reader client
// Creates a client for interacting with the document reader service
// Parameters:
// - cfg: Application configuration
//
// Returns:
// - Configured document reader client
// - Error if initialization fails
func initDocReaderClient(cfg *config.Config) (*client.Client, error) {
// Use the DocReader URL from environment or config
docReaderURL := os.Getenv("DOCREADER_ADDR")
if docReaderURL == "" && cfg.DocReader != nil {
docReaderURL = cfg.DocReader.Addr
// initDocReaderClient initializes the DocumentReader client (lightweight API).
func initDocReaderClient(cfg *config.Config) (interfaces.DocumentReader, error) {
addr := strings.TrimSpace(os.Getenv("DOCREADER_ADDR"))
transport := strings.TrimSpace(os.Getenv("DOCREADER_TRANSPORT"))
if transport == "" {
transport = "grpc"
}
return client.NewClient(docReaderURL)
if addr == "" {
logger.Infof(context.Background(), "[DocConverter] No DOCREADER_ADDR configured, starting disconnected")
}
transport = strings.ToLower(transport)
switch transport {
case "http", "https":
if addr != "" && !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") {
addr = "http://" + addr
}
return docparser.NewHTTPDocumentReader(addr)
default:
return docparser.NewGRPCDocumentReader(addr)
}
}
// initImageResolver creates the ImageResolver for storing images from DocReader.
func initImageResolver(cfg *config.Config) *docparser.ImageResolver {
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
}
urlPrefix := os.Getenv("LOCAL_STORAGE_URL_PREFIX")
if urlPrefix == "" {
urlPrefix = "/files"
}
return docparser.NewImageResolver(baseDir, urlPrefix)
}
// initOllamaService initializes the Ollama service client

View File

@ -4,10 +4,12 @@ import (
"context"
"fmt"
"os"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/database/sqlite3"
_ "github.com/golang-migrate/migrate/v4/source/file"
)
@ -30,8 +32,10 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error {
logger.Infof(ctx, "Starting database migration...")
// Use versioned migrations directory
migrationsPath := "file://migrations/versioned"
if strings.HasPrefix(dsn, "sqlite3://") {
migrationsPath = "file://migrations/sqlite"
}
m, err := migrate.New(migrationsPath, dsn)
if err != nil {

View File

@ -13,8 +13,6 @@ import (
"sync"
"time"
"github.com/Tencent/WeKnora/docreader/client"
"github.com/Tencent/WeKnora/docreader/proto"
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/errors"
@ -57,7 +55,7 @@ type InitializationHandler struct {
kbRepository interfaces.KnowledgeBaseRepository
knowledgeService interfaces.KnowledgeService
ollamaService *ollama.OllamaService
docReaderClient *client.Client
documentReader interfaces.DocumentReader
pooler embedding.EmbedderPooler
}
@ -70,7 +68,7 @@ func NewInitializationHandler(
kbRepository interfaces.KnowledgeBaseRepository,
knowledgeService interfaces.KnowledgeService,
ollamaService *ollama.OllamaService,
docReaderClient *client.Client,
documentReader interfaces.DocumentReader,
pooler embedding.EmbedderPooler,
) *InitializationHandler {
return &InitializationHandler{
@ -81,7 +79,7 @@ func NewInitializationHandler(
kbRepository: kbRepository,
knowledgeService: knowledgeService,
ollamaService: ollamaService,
docReaderClient: docReaderClient,
documentReader: documentReader,
pooler: pooler,
}
}
@ -94,30 +92,20 @@ type KBModelConfigRequest struct {
// 文档分块配置
DocumentSplitting struct {
ChunkSize int `json:"chunkSize"`
ChunkOverlap int `json:"chunkOverlap"`
Separators []string `json:"separators"`
ChunkSize int `json:"chunkSize"`
ChunkOverlap int `json:"chunkOverlap"`
Separators []string `json:"separators"`
ParserEngineRules []types.ParserEngineRule `json:"parserEngineRules,omitempty"`
} `json:"documentSplitting"`
// 多模态配置
// 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置)
Multimodal struct {
Enabled bool `json:"enabled"`
StorageType string `json:"storageType"` // "cos" or "minio"
COS *struct {
SecretID string `json:"secretId"`
SecretKey string `json:"secretKey"`
Region string `json:"region"`
BucketName string `json:"bucketName"`
AppID string `json:"appId"`
PathPrefix string `json:"pathPrefix"`
} `json:"cos"`
Minio *struct {
BucketName string `json:"bucketName"`
UseSSL bool `json:"useSSL"`
PathPrefix string `json:"pathPrefix"`
} `json:"minio"`
Enabled bool `json:"enabled"`
} `json:"multimodal"`
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
StorageProvider string `json:"storageProvider"`
// 知识图谱配置
NodeExtract struct {
Enabled bool `json:"enabled"`
@ -300,38 +288,22 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
if len(req.DocumentSplitting.Separators) > 0 {
kb.ChunkingConfig.Separators = req.DocumentSplitting.Separators
}
kb.ChunkingConfig.ParserEngineRules = req.DocumentSplitting.ParserEngineRules
// 更新多模态配置
if req.Multimodal.Enabled {
switch strings.ToLower(req.Multimodal.StorageType) {
case "cos":
if req.Multimodal.COS != nil {
kb.StorageConfig = types.StorageConfig{
SecretID: req.Multimodal.COS.SecretID,
SecretKey: req.Multimodal.COS.SecretKey,
Region: req.Multimodal.COS.Region,
BucketName: req.Multimodal.COS.BucketName,
AppID: req.Multimodal.COS.AppID,
PathPrefix: req.Multimodal.COS.PathPrefix,
Provider: "cos",
}
}
case "minio":
if req.Multimodal.Minio != nil {
kb.StorageConfig = types.StorageConfig{
BucketName: req.Multimodal.Minio.BucketName,
PathPrefix: req.Multimodal.Minio.PathPrefix,
Provider: "minio",
SecretID: os.Getenv("MINIO_ACCESS_KEY_ID"),
SecretKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"),
}
}
}
// VLM model already set above
} else {
// 多模态未启用时,清空存储配置
kb.StorageConfig = types.StorageConfig{}
kb.VLMConfig.ModelID = ""
}
// 存储引擎:仅写入 provider,参数从租户全局 StorageEngineConfig 读取
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
if provider == "" {
provider = "local"
}
kb.StorageConfig = types.StorageConfig{Provider: provider}
// 更新知识图谱配置
if req.NodeExtract.Enabled {
// 转换 Nodes 和 Relations 为指针类型
@ -1910,97 +1882,43 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {
})
}
// testMultimodalWithDocReader 调用docreader服务进行多模态处理
// testMultimodalWithDocReader uses DocumentReader.Read for document reading,
// then returns basic information about the result.
func (h *InitializationHandler) testMultimodalWithDocReader(
ctx context.Context,
imageContent []byte, filename string,
chunkSize, chunkOverlap int32, separators []string,
req *testMultimodalForm,
) (map[string]string, error) {
// 获取文件扩展名
fileExt := ""
if idx := strings.LastIndex(filename, "."); idx != -1 {
fileExt = strings.ToLower(filename[idx+1:])
}
// 检查docreader服务配置
if h.docReaderClient == nil {
if h.documentReader == nil {
return nil, fmt.Errorf("DocReader service not configured")
}
// 构造请求
request := &proto.ReadFromFileRequest{
requestID, _ := ctx.Value(types.RequestIDContextKey).(string)
readResult, err := h.documentReader.Read(ctx, &types.ReadRequest{
FileContent: imageContent,
FileName: filename,
FileType: fileExt,
ReadConfig: &proto.ReadConfig{
ChunkSize: chunkSize,
ChunkOverlap: chunkOverlap,
Separators: separators,
EnableMultimodal: true, // 启用多模态处理
VlmConfig: &proto.VLMConfig{
ModelName: req.VLMModel,
BaseUrl: req.VLMBaseURL,
ApiKey: req.VLMAPIKey,
InterfaceType: req.VLMInterfaceType,
},
},
RequestId: ctx.Value(types.RequestIDContextKey).(string),
}
// 设置对象存储配置(通用)
switch strings.ToLower(req.StorageType) {
case "cos":
request.ReadConfig.StorageConfig = &proto.StorageConfig{
Provider: proto.StorageProvider_COS,
Region: req.COSRegion,
BucketName: req.COSBucketName,
AccessKeyId: req.COSSecretID,
SecretAccessKey: req.COSSecretKey,
AppId: req.COSAppID,
PathPrefix: req.COSPathPrefix,
}
case "minio":
request.ReadConfig.StorageConfig = &proto.StorageConfig{
Provider: proto.StorageProvider_MINIO,
BucketName: req.MinioBucketName,
PathPrefix: req.MinioPathPrefix,
AccessKeyId: os.Getenv("MINIO_ACCESS_KEY_ID"),
SecretAccessKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"),
}
}
// 调用docreader服务
response, err := h.docReaderClient.ReadFromFile(ctx, request)
RequestID: requestID,
})
if err != nil {
return nil, fmt.Errorf("调用DocReader服务失败: %v", err)
}
if response.Error != "" {
return nil, fmt.Errorf("DocReader服务返回错误: %s", response.Error)
if readResult.Error != "" {
return nil, fmt.Errorf("DocReader服务返回错误: %s", readResult.Error)
}
// 处理响应,提取Caption和OCR信息
result := make(map[string]string)
var allCaptions, allOCRTexts []string
for _, chunk := range response.Chunks {
if len(chunk.Images) > 0 {
for _, image := range chunk.Images {
if image.Caption != "" {
allCaptions = append(allCaptions, image.Caption)
}
if image.OcrText != "" {
allOCRTexts = append(allOCRTexts, image.OcrText)
}
}
}
result := map[string]string{
"markdown": readResult.MarkdownContent,
"caption": "",
"ocr": "",
}
// 合并所有Caption和OCR结果
result["caption"] = strings.Join(allCaptions, "; ")
result["ocr"] = strings.Join(allOCRTexts, "; ")
return result, nil
}

View File

@ -738,6 +738,102 @@ func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) {
})
}
// mimeTypeByExt returns the MIME type for a given file extension.
func mimeTypeByExt(filename string) string {
ext := strings.ToLower(filename)
if idx := strings.LastIndex(ext, "."); idx >= 0 {
ext = ext[idx:]
} else {
ext = ""
}
m := map[string]string{
".pdf": "application/pdf",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".doc": "application/msword",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".ppt": "application/vnd.ms-powerpoint",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xls": "application/vnd.ms-excel",
".csv": "text/csv",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".bmp": "image/bmp",
".webp": "image/webp",
".svg": "image/svg+xml",
".tiff": "image/tiff",
".txt": "text/plain; charset=utf-8",
".md": "text/markdown; charset=utf-8",
".markdown": "text/markdown; charset=utf-8",
".json": "application/json; charset=utf-8",
".xml": "application/xml; charset=utf-8",
".html": "text/html; charset=utf-8",
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".ts": "text/typescript; charset=utf-8",
".py": "text/x-python; charset=utf-8",
".go": "text/x-go; charset=utf-8",
".java": "text/x-java; charset=utf-8",
".yaml": "text/yaml; charset=utf-8",
".yml": "text/yaml; charset=utf-8",
".sh": "text/x-shellscript; charset=utf-8",
}
if ct, ok := m[ext]; ok {
return ct
}
return "application/octet-stream"
}
// PreviewKnowledgeFile godoc
// @Summary 预览知识文件
// @Description 返回知识条目关联的原始文件,Content-Type 根据文件类型设置,用于浏览器内嵌预览
// @Tags 知识管理
// @Accept json
// @Produce application/pdf,image/jpeg,image/png,text/plain
// @Param id path string true "知识ID"
// @Success 200 {file} file "文件内容"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/preview [get]
func (h *KnowledgeHandler) PreviewKnowledgeFile(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleViewer)
if err != nil {
c.Error(err)
return
}
file, filename, err := h.kgService.GetKnowledgeFile(effCtx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error()))
return
}
defer file.Close()
contentType := mimeTypeByExt(filename)
c.Header("Content-Type", contentType)
c.Header("Content-Disposition", fmt.Sprintf("inline; filename=%s", filename))
c.Header("Cache-Control", "private, max-age=3600")
c.Stream(func(w io.Writer) bool {
if _, err := io.Copy(w, file); err != nil {
logger.Errorf(ctx, "Failed to stream preview: %v", err)
return false
}
return false
})
}
// GetKnowledgeBatchRequest defines parameters for batch knowledge retrieval
type GetKnowledgeBatchRequest struct {
IDs []string `form:"ids" binding:"required"` // List of knowledge IDs

Some files were not shown because too many files have changed in this diff Show More