feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.main
13
.env.example
|
|
@ -101,12 +101,7 @@ JWT_SECRET=weknora-jwt-secret
|
|||
# Embedding并发数,出现429错误时,可调小此参数
|
||||
CONCURRENCY_POOL_SIZE=5
|
||||
|
||||
# Docreader 并发任务数(图片OCR/Caption等异步任务),默认 1
|
||||
# 默认使用的 paddleocr 在高并发场景下,会出现异常,请谨慎设置
|
||||
# IMAGE_MAX_CONCURRENT=1
|
||||
|
||||
# Docreader OCR后端(no_ocr, paddle, nanonets)
|
||||
# OCR_BACKEND=paddle
|
||||
# (Removed: IMAGE_MAX_CONCURRENT, OCR_BACKEND — moved to Go App module after lightweight refactoring)
|
||||
|
||||
# 如果使用ElasticSearch作为向量存储,需要配置以下参数
|
||||
# ElasticSearch地址,例如 http://localhost:9200
|
||||
|
|
@ -242,3 +237,9 @@ APK_MIRROR_ARG=mirrors.tencent.com
|
|||
|
||||
# Milvus 数据库名称(可选)
|
||||
# MILVUS_DB_NAME=your_milvus_db_name
|
||||
|
||||
# Docreader 地址
|
||||
DOCREADER_ADDR=docreader:50051
|
||||
|
||||
# Docreader 连接方式
|
||||
DOCREADER_TRANSPORT=grpc
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
# WeKnora Lite 配置模板
|
||||
# 复制此文件为 .env.lite 并按需修改
|
||||
# cp .env.lite.example .env.lite
|
||||
|
||||
GIN_MODE=release
|
||||
|
||||
# === 数据库 ===
|
||||
DB_DRIVER=sqlite
|
||||
DB_PATH=./data/weknora.db
|
||||
|
||||
# === 检索引擎(FTS5 + sqlite-vec)===
|
||||
RETRIEVE_DRIVER=sqlite
|
||||
|
||||
# === 文件存储 ===
|
||||
STORAGE_TYPE=local
|
||||
LOCAL_STORAGE_BASE_DIR=./data/files
|
||||
|
||||
# === 流管理(内存,无 Redis)===
|
||||
STREAM_MANAGER_TYPE=memory
|
||||
|
||||
# === LLM 服务 ===
|
||||
# Ollama 本地服务(默认地址,按需修改)
|
||||
OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
# 如使用其他 OpenAI 兼容服务,取消注释:
|
||||
# OPENAI_API_KEY=sk-xxx
|
||||
# OPENAI_BASE_URL=https://api.openai.com/v1
|
||||
|
||||
# === 安全配置(生产环境请务必修改!)===
|
||||
TENANT_AES_KEY=CHANGE-ME-32-char-secret-key!!!!
|
||||
JWT_SECRET=CHANGE-ME-jwt-secret
|
||||
|
||||
# === 功能开关 ===
|
||||
NEO4J_ENABLE=false
|
||||
WEKNORA_SANDBOX_MODE=disabled
|
||||
ENABLE_GRAPH_RAG=false
|
||||
DISABLE_REGISTRATION=false
|
||||
|
||||
# === 性能 ===
|
||||
CONCURRENCY_POOL_SIZE=3
|
||||
|
||||
# Docreader 地址
|
||||
DOCREADER_ADDR=127.0.0.1:50051
|
||||
|
||||
# Docreader 传输方式
|
||||
DOCREADER_TRANSPORT=grpc
|
||||
|
|
@ -137,13 +137,13 @@ jobs:
|
|||
platform: linux/arm64
|
||||
runs: ubuntu-24.04-arm
|
||||
runs-on: ${{ matrix.runs }}
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
id: setup-buildx
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
|
|
@ -172,23 +172,6 @@ jobs:
|
|||
# 显示版本信息
|
||||
./scripts/get_version.sh info
|
||||
|
||||
- name: Build Cache for Docker
|
||||
uses: actions/cache@v4
|
||||
id: cache
|
||||
with:
|
||||
path: go-pkg-mod
|
||||
key: ${{ env.PLATFORM_PAIR }}-go-build-cache-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Inject go-build-cache
|
||||
uses: reproducible-containers/buildkit-cache-dance@v3
|
||||
with:
|
||||
builder: ${{ steps.setup-buildx.outputs.name }}
|
||||
cache-map: |
|
||||
{
|
||||
"go-pkg-mod": "/go/pkg/mod"
|
||||
}
|
||||
skip-extraction: ${{ steps.cache.outputs.cache-hit }}
|
||||
|
||||
- name: Build app Image
|
||||
id: build
|
||||
uses: docker/build-push-action@v3
|
||||
|
|
|
|||
|
|
@ -0,0 +1,246 @@
|
|||
name: Release Lite Binaries
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag (e.g. v0.2.0)"
|
||||
required: true
|
||||
|
||||
concurrency:
|
||||
group: release-lite-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.24"
|
||||
NODE_VERSION: "22"
|
||||
|
||||
jobs:
|
||||
# ── 1. Build frontend once, share as artifact ──
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Build frontend
|
||||
working-directory: frontend
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist/
|
||||
retention-days: 1
|
||||
|
||||
# ── 2. Build Go binary per platform (native CGO) ──
|
||||
build-binary:
|
||||
needs: build-frontend
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
- os: ubuntu-24.04-arm
|
||||
goos: linux
|
||||
goarch: arm64
|
||||
- os: macos-13
|
||||
goos: darwin
|
||||
goarch: amd64
|
||||
- os: macos-14
|
||||
goos: darwin
|
||||
goarch: arm64
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: true
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: web/
|
||||
|
||||
- name: Resolve version
|
||||
id: ver
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ]; then
|
||||
VERSION="${{ inputs.tag }}"
|
||||
else
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "Building ${VERSION} for ${{ matrix.goos }}/${{ matrix.goarch }}"
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
export EDITION=lite
|
||||
eval "$(./scripts/get_version.sh env)"
|
||||
LDFLAGS="-w -s $(./scripts/get_version.sh ldflags)"
|
||||
export CGO_CFLAGS="-Wno-deprecated-declarations"
|
||||
if [ "${{ matrix.goos }}" = "darwin" ]; then
|
||||
export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"
|
||||
fi
|
||||
go build -tags "sqlite_fts5" -ldflags="${LDFLAGS}" \
|
||||
-o WeKnora-lite ./cmd/server
|
||||
|
||||
- name: Package tarball
|
||||
run: |
|
||||
ARCHIVE="WeKnora-lite_${{ steps.ver.outputs.version }}_${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
mkdir -p "${ARCHIVE}/web"
|
||||
cp WeKnora-lite "${ARCHIVE}/"
|
||||
cp -r web/* "${ARCHIVE}/web/"
|
||||
cp .env.lite.example "${ARCHIVE}/"
|
||||
cp docs/LITE.md "${ARCHIVE}/README.md"
|
||||
if [ -d config ]; then
|
||||
cp -r config "${ARCHIVE}/config"
|
||||
fi
|
||||
if [ -d migrations/sqlite ]; then
|
||||
mkdir -p "${ARCHIVE}/migrations/sqlite"
|
||||
cp -r migrations/sqlite/* "${ARCHIVE}/migrations/sqlite/"
|
||||
fi
|
||||
if [ -f deploy/weknora-lite.service ]; then
|
||||
cp deploy/weknora-lite.service "${ARCHIVE}/"
|
||||
fi
|
||||
tar czf "${ARCHIVE}.tar.gz" "${ARCHIVE}"
|
||||
shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: release-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: |
|
||||
WeKnora-lite_*.tar.gz
|
||||
WeKnora-lite_*.tar.gz.sha256
|
||||
retention-days: 3
|
||||
|
||||
# ── 3. Create GitHub Release ──
|
||||
release:
|
||||
needs: build-binary
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: release-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Resolve version
|
||||
id: ver
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ]; then
|
||||
VERSION="${{ inputs.tag }}"
|
||||
else
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate release notes
|
||||
id: notes
|
||||
run: |
|
||||
cat > notes.md << 'NOTES'
|
||||
## WeKnora Lite ${{ steps.ver.outputs.version }}
|
||||
|
||||
单二进制部署,零外部依赖(无需 Docker / PostgreSQL / Redis)。
|
||||
|
||||
### 快速开始
|
||||
|
||||
```bash
|
||||
# 1. 解压
|
||||
tar xzf WeKnora-lite_${{ steps.ver.outputs.version }}_<os>_<arch>.tar.gz
|
||||
cd WeKnora-lite_${{ steps.ver.outputs.version }}_<os>_<arch>
|
||||
|
||||
# 2. 配置
|
||||
cp .env.lite.example .env.lite
|
||||
# 编辑 .env.lite,至少确认 OLLAMA_BASE_URL 正确
|
||||
|
||||
# 3. 启动 Ollama(如尚未运行)
|
||||
ollama serve &
|
||||
ollama pull qwen2.5:7b
|
||||
ollama pull nomic-embed-text
|
||||
|
||||
# 4. 运行
|
||||
set -a && source .env.lite && set +a
|
||||
./WeKnora-lite
|
||||
# 访问 http://localhost:8080
|
||||
```
|
||||
|
||||
### 平台支持
|
||||
|
||||
| 文件 | 平台 |
|
||||
|------|------|
|
||||
| `WeKnora-lite_*_linux_amd64.tar.gz` | Linux x86_64 |
|
||||
| `WeKnora-lite_*_linux_arm64.tar.gz` | Linux ARM64 |
|
||||
| `WeKnora-lite_*_darwin_amd64.tar.gz` | macOS Intel |
|
||||
| `WeKnora-lite_*_darwin_arm64.tar.gz` | macOS Apple Silicon |
|
||||
|
||||
详细文档见 [LITE.md](docs/LITE.md)。
|
||||
NOTES
|
||||
|
||||
- name: Create release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh release create "${{ steps.ver.outputs.version }}" \
|
||||
--title "WeKnora Lite ${{ steps.ver.outputs.version }}" \
|
||||
--notes-file notes.md \
|
||||
WeKnora-lite_*.tar.gz \
|
||||
WeKnora-lite_*.tar.gz.sha256
|
||||
|
||||
# ── 4. Update Homebrew Formula ──
|
||||
update-homebrew:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Resolve version
|
||||
id: ver
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ]; then
|
||||
VERSION="${{ inputs.tag }}"
|
||||
else
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Wait for release assets to be available
|
||||
run: sleep 15
|
||||
|
||||
- name: Update Formula
|
||||
run: ./scripts/update-homebrew-formula.sh "${{ steps.ver.outputs.version }}"
|
||||
|
||||
- name: Commit and push
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add Formula/weknora-lite.rb
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to Formula"
|
||||
else
|
||||
git commit -m "formula: update weknora-lite to ${{ steps.ver.outputs.version }}"
|
||||
git push origin main
|
||||
fi
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
# 忽略.env文件和其他包含敏感信息的配置文件
|
||||
.env
|
||||
.env.lite
|
||||
# 但不忽略示例文件
|
||||
!.env.example
|
||||
*.pem
|
||||
|
|
@ -27,9 +28,15 @@ logs/
|
|||
*.pid
|
||||
|
||||
WeKnora
|
||||
WeKnora-lite
|
||||
/models/
|
||||
test/data/mswag.txt
|
||||
data/files/
|
||||
data/weknora.db
|
||||
data/weknora.db-wal
|
||||
data/weknora.db-shm
|
||||
|
||||
web/
|
||||
|
||||
.venv/
|
||||
**/__pycache__
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
class WeknoraLite < Formula
|
||||
desc "Knowledge base management system — single-binary Lite edition"
|
||||
homepage "https://github.com/Tencent/WeKnora"
|
||||
version "0.0.0"
|
||||
license "Apache-2.0"
|
||||
|
||||
on_macos do
|
||||
on_arm do
|
||||
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_darwin_arm64.tar.gz"
|
||||
sha256 "PLACEHOLDER"
|
||||
end
|
||||
on_intel do
|
||||
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_darwin_amd64.tar.gz"
|
||||
sha256 "PLACEHOLDER"
|
||||
end
|
||||
end
|
||||
|
||||
on_linux do
|
||||
on_arm do
|
||||
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_linux_arm64.tar.gz"
|
||||
sha256 "PLACEHOLDER"
|
||||
end
|
||||
on_intel do
|
||||
url "https://github.com/Tencent/WeKnora/releases/download/v#{version}/WeKnora-lite_v#{version}_linux_amd64.tar.gz"
|
||||
sha256 "PLACEHOLDER"
|
||||
end
|
||||
end
|
||||
|
||||
def install
|
||||
libexec.install "WeKnora-lite"
|
||||
pkgshare.install "web" if File.directory?("web")
|
||||
pkgshare.install "config" if File.directory?("config")
|
||||
pkgshare.install ".env.lite.example"
|
||||
doc.install "README.md"
|
||||
pkgshare.install "migrations" if File.directory?("migrations")
|
||||
|
||||
(bin/"weknora-lite").write <<~SH
|
||||
#!/bin/bash
|
||||
CONFIG_DIR="${WEKNORA_CONFIG_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/weknora}"
|
||||
DATA_DIR="${WEKNORA_DATA_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/weknora}"
|
||||
|
||||
mkdir -p "$DATA_DIR/files" "$CONFIG_DIR/config" 2>/dev/null
|
||||
|
||||
if [ ! -f "$CONFIG_DIR/config/config.yaml" ]; then
|
||||
cp -r "#{pkgshare}/config/" "$CONFIG_DIR/config/"
|
||||
fi
|
||||
|
||||
if [ ! -d "$CONFIG_DIR/migrations" ] && [ -d "#{pkgshare}/migrations" ]; then
|
||||
ln -sf "#{pkgshare}/migrations" "$CONFIG_DIR/migrations"
|
||||
fi
|
||||
|
||||
if [ ! -f "$CONFIG_DIR/.env.lite" ]; then
|
||||
cp "#{pkgshare}/.env.lite.example" "$CONFIG_DIR/.env.lite"
|
||||
sed -i '' "s|DB_PATH=.*|DB_PATH=$DATA_DIR/weknora.db|" "$CONFIG_DIR/.env.lite"
|
||||
sed -i '' "s|LOCAL_STORAGE_BASE_DIR=.*|LOCAL_STORAGE_BASE_DIR=$DATA_DIR/files|" "$CONFIG_DIR/.env.lite"
|
||||
rm -f "$CONFIG_DIR/.env.lite-e"
|
||||
echo ""
|
||||
echo "已创建配置文件: $CONFIG_DIR/.env.lite"
|
||||
echo "请根据需要编辑(如修改 LLM 地址、安全密钥等)。"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
set -a
|
||||
source "$CONFIG_DIR/.env.lite"
|
||||
set +a
|
||||
|
||||
export DB_PATH="${DB_PATH:-$DATA_DIR/weknora.db}"
|
||||
export LOCAL_STORAGE_BASE_DIR="${LOCAL_STORAGE_BASE_DIR:-$DATA_DIR/files}"
|
||||
export WEKNORA_WEB_DIR="${WEKNORA_WEB_DIR:-#{pkgshare}/web}"
|
||||
|
||||
cd "$CONFIG_DIR"
|
||||
exec "#{libexec}/WeKnora-lite" "$@"
|
||||
SH
|
||||
end
|
||||
|
||||
def post_install
|
||||
(var/"weknora").mkpath
|
||||
(var/"log").mkpath
|
||||
end
|
||||
|
||||
service do
|
||||
run [bin/"weknora-lite"]
|
||||
keep_alive true
|
||||
working_dir var/"weknora"
|
||||
log_path var/"log/weknora-lite.log"
|
||||
error_log_path var/"log/weknora-lite.log"
|
||||
end
|
||||
|
||||
def caveats
|
||||
<<~EOS
|
||||
前台运行:
|
||||
weknora-lite
|
||||
|
||||
后台服务(推荐):
|
||||
brew services start weknora-lite # 启动并开机自启
|
||||
brew services stop weknora-lite # 停止
|
||||
brew services restart weknora-lite # 重启
|
||||
brew services info weknora-lite # 查看状态
|
||||
|
||||
日志:
|
||||
#{var}/log/weknora-lite.log
|
||||
|
||||
首次运行会自动创建配置文件:
|
||||
~/.config/weknora/.env.lite
|
||||
|
||||
数据存储在:
|
||||
~/.local/share/weknora/
|
||||
|
||||
如需修改配置(LLM 服务地址、安全密钥等):
|
||||
$EDITOR ~/.config/weknora/.env.lite
|
||||
brew services restart weknora-lite
|
||||
EOS
|
||||
end
|
||||
|
||||
test do
|
||||
assert_predicate bin/"weknora-lite", :executable?
|
||||
end
|
||||
end
|
||||
40
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger
|
||||
.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite
|
||||
|
||||
# Show help
|
||||
help:
|
||||
|
|
@ -56,6 +56,11 @@ help:
|
|||
@echo " dev-status 查看开发环境状态"
|
||||
@echo " dev-app 启动后端应用(本地运行,需先运行 dev-start)"
|
||||
@echo " dev-frontend 启动前端(本地运行,需先运行 dev-start)"
|
||||
@echo ""
|
||||
@echo "Lite 模式(零外部依赖):"
|
||||
@echo " build-lite 构建 Lite 版本(先构建前端到 web/,再构建 Go;SKIP_FRONTEND=1 跳过前端)"
|
||||
@echo " run-lite 构建并启动 Lite 版本"
|
||||
@echo " package-lite 构建并打包 Lite 发行包(tarball)"
|
||||
|
||||
# Go related variables
|
||||
BINARY_NAME=WeKnora
|
||||
|
|
@ -223,11 +228,42 @@ deps:
|
|||
build-prod:
|
||||
VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo "$${VERSION:-unknown}"); \
|
||||
COMMIT_ID=$${COMMIT_ID:-unknown}; \
|
||||
CGO_ENABLED=1 \
|
||||
CGO_CFLAGS="-Wno-deprecated-declarations" \
|
||||
CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \
|
||||
BUILD_TIME=$${BUILD_TIME:-unknown}; \
|
||||
GO_VERSION=$${GO_VERSION:-unknown}; \
|
||||
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
|
||||
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=standard' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
|
||||
go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
|
||||
# Build Lite version (single binary, SQLite + in-memory queue)
|
||||
# 会先构建前端到 web/,再构建 Go 二进制;SKIP_FRONTEND=1 可跳过前端
|
||||
build-lite:
|
||||
@if [ -f frontend/package.json ] && [ "$${SKIP_FRONTEND:-}" != "1" ]; then \
|
||||
echo ">> Building frontend for Lite..."; \
|
||||
(cd frontend && npm ci --prefer-offline && npm run build) && \
|
||||
rm -rf web && cp -r frontend/dist web; \
|
||||
elif [ "$${SKIP_FRONTEND:-}" = "1" ]; then \
|
||||
echo ">> Skipping frontend (SKIP_FRONTEND=1)"; \
|
||||
else \
|
||||
echo ">> No frontend/package.json, skipping frontend"; \
|
||||
fi
|
||||
EDITION=lite eval "$$(./scripts/get_version.sh env)"; \
|
||||
LDFLAGS="$$(EDITION=lite ./scripts/get_version.sh ldflags)"; \
|
||||
CGO_ENABLED=1 \
|
||||
CGO_CFLAGS="-Wno-deprecated-declarations" \
|
||||
CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \
|
||||
go build -tags "sqlite_fts5" -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME)-lite $(MAIN_PATH)
|
||||
|
||||
# Run Lite version with .env.lite defaults
|
||||
run-lite: build-lite
|
||||
@if [ ! -f .env.lite ]; then echo "Error: .env.lite not found"; exit 1; fi
|
||||
@set -a && . ./.env.lite && set +a && ./$(BINARY_NAME)-lite
|
||||
|
||||
# Package Lite version into distributable tarball
|
||||
package-lite:
|
||||
./scripts/package-lite.sh
|
||||
|
||||
download_spatial:
|
||||
go run cmd/download/duckdb/duckdb.go
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
[Unit]
|
||||
Description=WeKnora Lite - Knowledge Base Management System
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=weknora
|
||||
Group=weknora
|
||||
WorkingDirectory=/opt/weknora
|
||||
EnvironmentFile=/opt/weknora/.env.lite
|
||||
ExecStart=/opt/weknora/WeKnora-lite
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
LimitNOFILE=65536
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/weknora/data
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -107,16 +107,19 @@ services:
|
|||
restart: "no"
|
||||
|
||||
docreader:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile.docreader
|
||||
image: wechatopenai/weknora-docreader:latest
|
||||
container_name: WeKnora-docreader-dev
|
||||
ports:
|
||||
- "${DOCREADER_PORT:-50051}:50051"
|
||||
volumes:
|
||||
- docreader-tmp-dev:/tmp/docreader
|
||||
environment:
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- STORAGE_TYPE=${STORAGE_TYPE:-}
|
||||
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
|
||||
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
|
||||
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
|
||||
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}}
|
||||
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
|
||||
healthcheck:
|
||||
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
|
||||
interval: 30s
|
||||
|
|
@ -165,4 +168,5 @@ volumes:
|
|||
neo4j-data-dev:
|
||||
jaeger_data_dev:
|
||||
qdrant_data_dev:
|
||||
docreader-tmp-dev:
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ services:
|
|||
- "${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- data-files:/data/files
|
||||
- docreader-tmp:/tmp/docreader:ro
|
||||
# Optional: mount custom config file
|
||||
# - ./config/config.yaml:/app/config/config.yaml
|
||||
# Optional: mount custom skills directory (allows adding skills without rebuilding image)
|
||||
|
|
@ -83,7 +84,8 @@ services:
|
|||
- QDRANT_USE_TLS=${QDRANT_USE_TLS:-false}
|
||||
- MILVUS_ADDRESS=milvus:19530
|
||||
- MILVUS_COLLECTION=${MILVUS_COLLECTION:-weknora_embeddings}
|
||||
- DOCREADER_ADDR=docreader:50051
|
||||
- DOCREADER_ADDR=${DOCREADER_ADDR:-docreader:50051}
|
||||
- DOCREADER_TRANSPORT=${DOCREADER_TRANSPORT:-grpc}
|
||||
- STORAGE_TYPE=${STORAGE_TYPE:-}
|
||||
- LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-}
|
||||
- AUTO_RECOVER_DIRTY=${AUTO_RECOVER_DIRTY:-true}
|
||||
|
|
@ -146,10 +148,10 @@ services:
|
|||
container_name: WeKnora-docreader
|
||||
ports:
|
||||
- "${DOCREADER_PORT:-50051}:50051"
|
||||
volumes:
|
||||
- docreader-tmp:/tmp/docreader
|
||||
environment:
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
|
||||
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
|
||||
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
|
||||
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
|
||||
healthcheck:
|
||||
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
|
||||
|
|
@ -312,6 +314,7 @@ networks:
|
|||
volumes:
|
||||
postgres-data:
|
||||
data-files:
|
||||
docreader-tmp:
|
||||
jaeger_data:
|
||||
minio_data:
|
||||
neo4j-data:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# =========================
|
||||
# 构建阶段
|
||||
# 构建阶段(轻量化:仅文档解析 + 图片提取,无 OCR/VLM)
|
||||
# =========================
|
||||
FROM python:3.10.18-bookworm AS builder
|
||||
|
||||
|
|
@ -15,7 +15,6 @@ RUN apt-get update && apt-get install -y \
|
|||
python3-dev \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
libpq-dev \
|
||||
libffi-dev \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
|
|
@ -25,11 +24,10 @@ RUN apt-get update && apt-get install -y \
|
|||
unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 检查是否存在本地protoc安装包,如果存在则离线安装,否则在线安装,其他安装包按需求添加
|
||||
# 检查是否存在本地protoc安装包,如果存在则离线安装,否则在线安装
|
||||
ARG TARGETARCH
|
||||
COPY packages/ /app/packages/
|
||||
RUN echo "检查本地protoc安装包..." && \
|
||||
# 根据目标架构选择正确的protoc包名
|
||||
case ${TARGETARCH} in \
|
||||
"amd64") PROTOC_ARCH="x86_64" ;; \
|
||||
"arm64") PROTOC_ARCH="aarch_64" ;; \
|
||||
|
|
@ -39,41 +37,18 @@ RUN echo "检查本地protoc安装包..." && \
|
|||
PROTOC_PACKAGE="protoc-3.19.4-linux-${PROTOC_ARCH}.zip" && \
|
||||
if [ -f "/app/packages/${PROTOC_PACKAGE}" ]; then \
|
||||
echo "发现本地protoc安装包,将进行离线安装"; \
|
||||
# 离线安装:使用本地包(精确路径避免歧义)
|
||||
cp /app/packages/${PROTOC_PACKAGE} /app/ && \
|
||||
unzip -o /app/${PROTOC_PACKAGE} -d /usr/local && \
|
||||
chmod +x /usr/local/bin/protoc && \
|
||||
rm -f /app/${PROTOC_PACKAGE}; \
|
||||
else \
|
||||
echo "未发现本地protoc安装包,将进行在线安装"; \
|
||||
# 在线安装:从网络下载
|
||||
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/${PROTOC_PACKAGE} && \
|
||||
unzip -o ${PROTOC_PACKAGE} -d /usr/local && \
|
||||
chmod +x /usr/local/bin/protoc && \
|
||||
rm -f ${PROTOC_PACKAGE}; \
|
||||
fi
|
||||
|
||||
# 预下载 PP-OCRv4 模型
|
||||
RUN mkdir -p /root/.paddleocr/whl/det/ch && \
|
||||
mkdir -p /root/.paddleocr/whl/rec/ch && \
|
||||
mkdir -p /root/.paddleocr/whl/cls/ch && \
|
||||
# 下载检测模型
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_det_infer.tar \
|
||||
-O /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
|
||||
tar -xf /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar -C /root/.paddleocr/whl/det/ch/ && \
|
||||
# 下载识别模型
|
||||
wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar \
|
||||
-O /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
|
||||
tar -xf /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar -C /root/.paddleocr/whl/rec/ch/ && \
|
||||
# 下载文本方向分类模型(用于判断文本是否需要旋转)
|
||||
wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar \
|
||||
-O /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar && \
|
||||
tar -xf /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar -C /root/.paddleocr/whl/cls/ && \
|
||||
# 清理压缩包
|
||||
rm -f /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \
|
||||
rm -f /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \
|
||||
rm -f /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar
|
||||
|
||||
# 复制依赖文件
|
||||
COPY docreader/pyproject.toml docreader/uv.lock ./
|
||||
RUN pip install uv --break-system-packages && \
|
||||
|
|
@ -82,15 +57,13 @@ RUN pip install uv --break-system-packages && \
|
|||
# 复制源代码和生成脚本
|
||||
COPY docreader docreader
|
||||
|
||||
# 生成 protobuf 代码
|
||||
# 生成 protobuf 代码(使用 venv 中的 grpc_tools)
|
||||
ENV PATH="/app/.venv/bin:${PATH}"
|
||||
RUN chmod +x docreader/scripts/generate_proto.sh && \
|
||||
bash docreader/scripts/generate_proto.sh
|
||||
|
||||
# 确保模型目录存在
|
||||
RUN ls -la /root/.paddleocr/whl/
|
||||
|
||||
# =========================
|
||||
# 运行阶段
|
||||
# 运行阶段(轻量化)
|
||||
# =========================
|
||||
FROM python:3.10.18-bookworm AS runner
|
||||
|
||||
|
|
@ -100,16 +73,14 @@ RUN sed -i 's@http://deb.debian.org@https://mirrors.tuna.tsinghua.edu.cn@g' /etc
|
|||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装运行时依赖
|
||||
# 安装运行时依赖(已移除 OCR/PaddleOCR 相关依赖)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
libjpeg62-turbo \
|
||||
libpq5 \
|
||||
wget \
|
||||
gnupg \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
antiword \
|
||||
vim \
|
||||
tar \
|
||||
dpkg \
|
||||
libxinerama1 \
|
||||
|
|
@ -126,7 +97,6 @@ RUN apt-get update && apt-get install -y \
|
|||
# 安装 grpc_health_probe
|
||||
ARG TARGETARCH
|
||||
RUN GRPC_HEALTH_PROBE_VERSION=v0.4.24 && \
|
||||
# 根据目标架构选择正确的二进制文件
|
||||
case ${TARGETARCH} in \
|
||||
"amd64") ARCH="amd64" ;; \
|
||||
"arm64") ARCH="arm64" ;; \
|
||||
|
|
@ -135,25 +105,24 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.24 && \
|
|||
esac && \
|
||||
wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${ARCH} && \
|
||||
chmod +x /bin/grpc_health_probe
|
||||
|
||||
|
||||
# 从构建阶段复制已安装的依赖和生成的代码
|
||||
ENV VIRTUAL_ENV=/app/.venv
|
||||
COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
|
||||
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
|
||||
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
COPY --from=builder /root/.paddleocr /root/.paddleocr
|
||||
|
||||
# 安装 Playwright 浏览器
|
||||
# 安装 Playwright 浏览器(网页解析)
|
||||
RUN python -m playwright install webkit
|
||||
RUN python -m playwright install-deps webkit
|
||||
|
||||
# COPY docreader/scripts/download_deps.py download_deps.py
|
||||
# RUN python -m download_deps
|
||||
|
||||
COPY docreader/pyproject.toml docreader/uv.lock ./
|
||||
COPY --from=builder /app/docreader docreader
|
||||
|
||||
# 创建共享临时图片目录
|
||||
RUN mkdir -p /tmp/docreader
|
||||
|
||||
# 暴露 gRPC 端口
|
||||
EXPOSE 50051
|
||||
|
||||
|
|
|
|||
|
|
@ -13,42 +13,35 @@ import (
|
|||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
// getMaxMessageSize returns the maximum gRPC message size in bytes.
|
||||
// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable.
|
||||
func getMaxMessageSize() int {
|
||||
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
|
||||
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
|
||||
return size * 1024 * 1024
|
||||
}
|
||||
}
|
||||
return 50 * 1024 * 1024 // default 50MB
|
||||
return 50 * 1024 * 1024
|
||||
}
|
||||
|
||||
// Logger is the default logger used by the client
|
||||
var Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
|
||||
|
||||
// ImageInfo 表示一个图片的信息
|
||||
type ImageInfo struct {
|
||||
URL string // 图片URL(COS)
|
||||
Caption string // 图片描述
|
||||
OCRText string // OCR提取的文本
|
||||
OriginalURL string // 原始图片URL
|
||||
Start int // 图片在文本中的开始位置
|
||||
End int // 图片在文本中的结束位置
|
||||
// ImageRefInfo represents an image reference from a converted document.
|
||||
type ImageRefInfo struct {
|
||||
Filename string
|
||||
OriginalRef string
|
||||
MimeType string
|
||||
StorageKey string
|
||||
}
|
||||
|
||||
// Client represents a DocReader service client
|
||||
// Client represents a DocReader service client.
|
||||
type Client struct {
|
||||
conn *grpc.ClientConn
|
||||
proto.DocReaderClient
|
||||
debug bool
|
||||
}
|
||||
|
||||
// NewClient creates a new DocReader client with the specified address
|
||||
func NewClient(addr string) (*Client, error) {
|
||||
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
|
||||
|
||||
// 设置消息大小限制 (configurable via GRPC_MAX_MESSAGE_SIZE_MB)
|
||||
maxMsgSize := getMaxMessageSize()
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
|
|
@ -75,19 +68,15 @@ func NewClient(addr string) (*Client, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the client connection
|
||||
func (c *Client) Close() error {
|
||||
Logger.Printf("INFO: Closing DocReader client connection")
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SetDebug enables or disables debug logging
|
||||
func (c *Client) SetDebug(debug bool) {
|
||||
c.debug = debug
|
||||
Logger.Printf("INFO: Debug logging set to %v", debug)
|
||||
}
|
||||
|
||||
// Log logs a message with the appropriate level
|
||||
func (c *Client) Log(level string, format string, args ...interface{}) {
|
||||
if level == "DEBUG" && !c.debug {
|
||||
return
|
||||
|
|
@ -95,28 +84,20 @@ func (c *Client) Log(level string, format string, args ...interface{}) {
|
|||
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
|
||||
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
|
||||
if chunk == nil || len(chunk.Images) == 0 {
|
||||
// GetImageRefsFromResponse extracts image references from a ReadResponse.
|
||||
func GetImageRefsFromResponse(resp *proto.ReadResponse) []ImageRefInfo {
|
||||
if resp == nil || len(resp.ImageRefs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
images := make([]ImageInfo, 0, len(chunk.Images))
|
||||
for _, img := range chunk.Images {
|
||||
images = append(images, ImageInfo{
|
||||
URL: img.Url,
|
||||
Caption: img.Caption,
|
||||
OCRText: img.OcrText,
|
||||
OriginalURL: img.OriginalUrl,
|
||||
Start: int(img.Start),
|
||||
End: int(img.End),
|
||||
refs := make([]ImageRefInfo, 0, len(resp.ImageRefs))
|
||||
for _, ref := range resp.ImageRefs {
|
||||
refs = append(refs, ImageRefInfo{
|
||||
Filename: ref.Filename,
|
||||
OriginalRef: ref.OriginalRef,
|
||||
MimeType: ref.MimeType,
|
||||
StorageKey: ref.StorageKey,
|
||||
})
|
||||
}
|
||||
|
||||
return images
|
||||
}
|
||||
|
||||
// HasImagesInChunk 判断一个Chunk是否包含图片
|
||||
func HasImagesInChunk(chunk *proto.Chunk) bool {
|
||||
return chunk != nil && len(chunk.Images) > 0
|
||||
return refs
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,144 +11,75 @@ import (
|
|||
)
|
||||
|
||||
func init() {
|
||||
// 配置测试日志
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
|
||||
log.Println("INFO: Initializing DocReader client tests")
|
||||
}
|
||||
|
||||
func TestReadFromURL(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromURL")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
func TestReadURL(t *testing.T) {
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 测试 ReadFromURL 方法
|
||||
log.Println("INFO: Sending ReadFromURL request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromURL(
|
||||
resp, err := client.Read(
|
||||
context.Background(),
|
||||
&proto.ReadFromURLRequest{
|
||||
&proto.ReadRequest{
|
||||
Url: "https://example.com",
|
||||
Title: "test",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 512,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
log.Printf("INFO: Read(URL) completed in %v", time.Since(startTime))
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromURL failed: %v", err)
|
||||
t.Fatalf("ReadFromURL failed: %v", err)
|
||||
t.Fatalf("Read failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
if resp.Error != "" {
|
||||
t.Fatalf("Read returned error: %s", resp.Error)
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
if resp.MarkdownContent == "" {
|
||||
t.Error("Expected non-empty markdown content")
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromURL completed successfully")
|
||||
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(resp.ImageRefs))
|
||||
}
|
||||
|
||||
func TestReadFromFileWithChunking(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromFileWithChunking")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
func TestReadFile(t *testing.T) {
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 读取测试文件
|
||||
log.Println("INFO: Reading test file")
|
||||
fileContent, err := os.ReadFile("../testdata/test.md")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to read test file: %v", err)
|
||||
t.Fatalf("Failed to read test file: %v", err)
|
||||
}
|
||||
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
|
||||
|
||||
// 测试 ReadFromFile 方法,带分块参数
|
||||
log.Println("INFO: Sending ReadFromFile request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromFile(
|
||||
resp, err := client.Read(
|
||||
context.Background(),
|
||||
&proto.ReadFromFileRequest{
|
||||
&proto.ReadRequest{
|
||||
FileContent: fileContent,
|
||||
FileName: "test.md",
|
||||
FileType: "md",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 200,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
log.Printf("INFO: Read(file) completed in %v", time.Since(startTime))
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromFile failed: %v", err)
|
||||
t.Fatalf("ReadFromFile failed: %v", err)
|
||||
t.Fatalf("Read failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
if resp.Error != "" {
|
||||
t.Fatalf("Read returned error: %s", resp.Error)
|
||||
}
|
||||
if resp.MarkdownContent == "" {
|
||||
t.Error("Expected non-empty markdown content")
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
|
||||
}
|
||||
|
||||
// 截断字符串以供日志打印
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
imageRefs := GetImageRefsFromResponse(resp)
|
||||
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(imageRefs))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,58 +52,17 @@ class DocReaderConfig:
|
|||
grpc_max_file_size_mb: int
|
||||
grpc_port: int
|
||||
|
||||
# Image processing
|
||||
image_max_concurrent: int
|
||||
|
||||
# Proxy
|
||||
external_http_proxy: str
|
||||
external_https_proxy: str
|
||||
|
||||
# OCR
|
||||
ocr_backend: str
|
||||
ocr_api_base_url: str
|
||||
ocr_api_key: str
|
||||
ocr_model: str
|
||||
|
||||
# VLM Caption
|
||||
vlm_model_base_url: str
|
||||
vlm_model_name: str
|
||||
vlm_model_api_key: str
|
||||
vlm_interface_type: str
|
||||
|
||||
# Storage
|
||||
storage_type: str
|
||||
|
||||
cos_secret_id: str
|
||||
cos_secret_key: str
|
||||
cos_region: str
|
||||
cos_bucket_name: str
|
||||
cos_app_id: str
|
||||
cos_path_prefix: str
|
||||
cos_enable_old_domain: bool
|
||||
|
||||
minio_access_key_id: str
|
||||
minio_secret_access_key: str
|
||||
minio_bucket_name: str
|
||||
minio_path_prefix: str
|
||||
minio_endpoint: str
|
||||
minio_public_endpoint: str
|
||||
minio_use_ssl: bool
|
||||
|
||||
local_storage_base_dir: str
|
||||
|
||||
# Other
|
||||
mineru_endpoint: str
|
||||
# Temp image output directory (shared with Go app via volume, local mode fallback)
|
||||
image_output_dir: str
|
||||
|
||||
|
||||
def load_config() -> DocReaderConfig:
|
||||
"""Load config from environment variables.
|
||||
"""Load config from environment variables (lightweight version)."""
|
||||
|
||||
Naming convention (new): DOCREADER_*
|
||||
Backward compatible keys are supported.
|
||||
"""
|
||||
|
||||
# gRPC
|
||||
grpc_max_workers = _get_int(["DOCREADER_GRPC_MAX_WORKERS", "GRPC_MAX_WORKERS"], 4)
|
||||
grpc_max_file_size_mb = (
|
||||
_get_int(["DOCREADER_GRPC_MAX_FILE_SIZE_MB", "MAX_FILE_SIZE_MB"], 50)
|
||||
|
|
@ -112,12 +71,6 @@ def load_config() -> DocReaderConfig:
|
|||
)
|
||||
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
|
||||
|
||||
# Image processing
|
||||
image_max_concurrent = _get_int(
|
||||
["DOCREADER_IMAGE_MAX_CONCURRENT", "IMAGE_MAX_CONCURRENT"], 1
|
||||
)
|
||||
|
||||
# Proxies
|
||||
external_http_proxy = _get_str(
|
||||
["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], ""
|
||||
)
|
||||
|
|
@ -125,95 +78,17 @@ def load_config() -> DocReaderConfig:
|
|||
["DOCREADER_EXTERNAL_HTTPS_PROXY", "EXTERNAL_HTTPS_PROXY"], ""
|
||||
)
|
||||
|
||||
# OCR
|
||||
ocr_backend = _get_str(["DOCREADER_OCR_BACKEND", "OCR_BACKEND"], "")
|
||||
ocr_api_base_url = _get_str(["DOCREADER_OCR_API_BASE_URL", "OCR_API_BASE_URL"], "")
|
||||
ocr_api_key = _get_str(["DOCREADER_OCR_API_KEY", "OCR_API_KEY"], "")
|
||||
ocr_model = _get_str(["DOCREADER_OCR_MODEL", "OCR_MODEL"], "")
|
||||
|
||||
# VLM Caption
|
||||
vlm_model_base_url = _get_str(
|
||||
["DOCREADER_VLM_MODEL_BASE_URL", "VLM_MODEL_BASE_URL"], ""
|
||||
image_output_dir = _get_str(
|
||||
["DOCREADER_IMAGE_OUTPUT_DIR", "IMAGE_OUTPUT_DIR"], "/tmp/docreader"
|
||||
)
|
||||
vlm_model_name = _get_str(["DOCREADER_VLM_MODEL_NAME", "VLM_MODEL_NAME"], "")
|
||||
vlm_model_api_key = _get_str(
|
||||
["DOCREADER_VLM_MODEL_API_KEY", "VLM_MODEL_API_KEY"], ""
|
||||
)
|
||||
vlm_interface_type = _get_str(
|
||||
["DOCREADER_VLM_INTERFACE_TYPE", "VLM_INTERFACE_TYPE"], "openai"
|
||||
).lower()
|
||||
|
||||
# Storage
|
||||
storage_type = _get_str(["DOCREADER_STORAGE_TYPE", "STORAGE_TYPE"], "cos").lower()
|
||||
|
||||
# COS
|
||||
cos_secret_id = _get_str(["DOCREADER_COS_SECRET_ID", "COS_SECRET_ID"], "")
|
||||
cos_secret_key = _get_str(["DOCREADER_COS_SECRET_KEY", "COS_SECRET_KEY"], "")
|
||||
cos_region = _get_str(["DOCREADER_COS_REGION", "COS_REGION"], "")
|
||||
cos_bucket_name = _get_str(["DOCREADER_COS_BUCKET_NAME", "COS_BUCKET_NAME"], "")
|
||||
cos_app_id = _get_str(["DOCREADER_COS_APP_ID", "COS_APP_ID"], "")
|
||||
cos_path_prefix = _get_str(["DOCREADER_COS_PATH_PREFIX", "COS_PATH_PREFIX"], "")
|
||||
cos_enable_old_domain = _get_bool(
|
||||
["DOCREADER_COS_ENABLE_OLD_DOMAIN", "COS_ENABLE_OLD_DOMAIN"], True
|
||||
)
|
||||
|
||||
# MinIO
|
||||
minio_access_key_id = _get_str(
|
||||
["DOCREADER_MINIO_ACCESS_KEY_ID", "MINIO_ACCESS_KEY_ID"], "minioadmin"
|
||||
)
|
||||
minio_secret_access_key = _get_str(
|
||||
["DOCREADER_MINIO_SECRET_ACCESS_KEY", "MINIO_SECRET_ACCESS_KEY"], "minioadmin"
|
||||
)
|
||||
minio_bucket_name = _get_str(
|
||||
["DOCREADER_MINIO_BUCKET_NAME", "MINIO_BUCKET_NAME"], "WeKnora"
|
||||
)
|
||||
minio_path_prefix = _get_str(
|
||||
["DOCREADER_MINIO_PATH_PREFIX", "MINIO_PATH_PREFIX"], ""
|
||||
)
|
||||
minio_endpoint = _get_str(["DOCREADER_MINIO_ENDPOINT", "MINIO_ENDPOINT"], "")
|
||||
minio_public_endpoint = _get_str(
|
||||
["DOCREADER_MINIO_PUBLIC_ENDPOINT", "MINIO_PUBLIC_ENDPOINT"], ""
|
||||
)
|
||||
minio_use_ssl = _get_bool(["DOCREADER_MINIO_USE_SSL", "MINIO_USE_SSL"], False)
|
||||
|
||||
# Local storage
|
||||
local_storage_base_dir = "./data/files"
|
||||
|
||||
# Other
|
||||
mineru_endpoint = _get_str(["DOCREADER_MINERU_ENDPOINT", "MINERU_ENDPOINT"], "")
|
||||
|
||||
return DocReaderConfig(
|
||||
grpc_max_workers=grpc_max_workers,
|
||||
grpc_max_file_size_mb=grpc_max_file_size_mb,
|
||||
grpc_port=grpc_port,
|
||||
image_max_concurrent=image_max_concurrent,
|
||||
external_http_proxy=external_http_proxy,
|
||||
external_https_proxy=external_https_proxy,
|
||||
ocr_backend=ocr_backend,
|
||||
ocr_api_base_url=ocr_api_base_url,
|
||||
ocr_api_key=ocr_api_key,
|
||||
ocr_model=ocr_model,
|
||||
vlm_model_base_url=vlm_model_base_url,
|
||||
vlm_model_name=vlm_model_name,
|
||||
vlm_model_api_key=vlm_model_api_key,
|
||||
vlm_interface_type=vlm_interface_type,
|
||||
storage_type=storage_type,
|
||||
cos_secret_id=cos_secret_id,
|
||||
cos_secret_key=cos_secret_key,
|
||||
cos_region=cos_region,
|
||||
cos_bucket_name=cos_bucket_name,
|
||||
cos_app_id=cos_app_id,
|
||||
cos_path_prefix=cos_path_prefix,
|
||||
cos_enable_old_domain=cos_enable_old_domain,
|
||||
minio_access_key_id=minio_access_key_id,
|
||||
minio_secret_access_key=minio_secret_access_key,
|
||||
minio_bucket_name=minio_bucket_name,
|
||||
minio_path_prefix=minio_path_prefix,
|
||||
minio_endpoint=minio_endpoint,
|
||||
minio_public_endpoint=minio_public_endpoint,
|
||||
minio_use_ssl=minio_use_ssl,
|
||||
local_storage_base_dir=local_storage_base_dir,
|
||||
mineru_endpoint=mineru_endpoint,
|
||||
image_output_dir=image_output_dir,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -223,56 +98,12 @@ CONFIG = load_config()
|
|||
def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
|
||||
cfg = CONFIG
|
||||
d: Dict[str, Any] = {
|
||||
# gRPC
|
||||
"DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers,
|
||||
"DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb,
|
||||
"DOCREADER_GRPC_PORT": cfg.grpc_port,
|
||||
# Image processing
|
||||
"DOCREADER_IMAGE_MAX_CONCURRENT": cfg.image_max_concurrent,
|
||||
# Proxy
|
||||
"DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy,
|
||||
"DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy,
|
||||
# OCR
|
||||
"DOCREADER_OCR_BACKEND": cfg.ocr_backend,
|
||||
"DOCREADER_OCR_API_BASE_URL": cfg.ocr_api_base_url,
|
||||
"DOCREADER_OCR_API_KEY": _mask_secret(cfg.ocr_api_key)
|
||||
if mask_secrets
|
||||
else cfg.ocr_api_key,
|
||||
"DOCREADER_OCR_MODEL": cfg.ocr_model,
|
||||
# VLM
|
||||
"DOCREADER_VLM_MODEL_BASE_URL": cfg.vlm_model_base_url,
|
||||
"DOCREADER_VLM_MODEL_NAME": cfg.vlm_model_name,
|
||||
"DOCREADER_VLM_MODEL_API_KEY": _mask_secret(cfg.vlm_model_api_key)
|
||||
if mask_secrets
|
||||
else cfg.vlm_model_api_key,
|
||||
"DOCREADER_VLM_INTERFACE_TYPE": cfg.vlm_interface_type,
|
||||
# Storage
|
||||
"DOCREADER_STORAGE_TYPE": cfg.storage_type,
|
||||
"DOCREADER_COS_SECRET_ID": _mask_secret(cfg.cos_secret_id)
|
||||
if mask_secrets
|
||||
else cfg.cos_secret_id,
|
||||
"DOCREADER_COS_SECRET_KEY": _mask_secret(cfg.cos_secret_key)
|
||||
if mask_secrets
|
||||
else cfg.cos_secret_key,
|
||||
"DOCREADER_COS_REGION": cfg.cos_region,
|
||||
"DOCREADER_COS_BUCKET_NAME": cfg.cos_bucket_name,
|
||||
"DOCREADER_COS_APP_ID": cfg.cos_app_id,
|
||||
"DOCREADER_COS_PATH_PREFIX": cfg.cos_path_prefix,
|
||||
"DOCREADER_COS_ENABLE_OLD_DOMAIN": cfg.cos_enable_old_domain,
|
||||
"DOCREADER_MINIO_ACCESS_KEY_ID": _mask_secret(cfg.minio_access_key_id)
|
||||
if mask_secrets
|
||||
else cfg.minio_access_key_id,
|
||||
"DOCREADER_MINIO_SECRET_ACCESS_KEY": _mask_secret(cfg.minio_secret_access_key)
|
||||
if mask_secrets
|
||||
else cfg.minio_secret_access_key,
|
||||
"DOCREADER_MINIO_BUCKET_NAME": cfg.minio_bucket_name,
|
||||
"DOCREADER_MINIO_PATH_PREFIX": cfg.minio_path_prefix,
|
||||
"DOCREADER_MINIO_ENDPOINT": cfg.minio_endpoint,
|
||||
"DOCREADER_MINIO_PUBLIC_ENDPOINT": cfg.minio_public_endpoint,
|
||||
"DOCREADER_MINIO_USE_SSL": cfg.minio_use_ssl,
|
||||
"DOCREADER_LOCAL_STORAGE_BASE_DIR": cfg.local_storage_base_dir,
|
||||
# Other
|
||||
"DOCREADER_MINERU_ENDPOINT": cfg.mineru_endpoint,
|
||||
"DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir,
|
||||
}
|
||||
return d
|
||||
|
||||
|
|
|
|||
|
|
@ -13,118 +13,130 @@ from grpc_health.v1.health import HealthServicer
|
|||
|
||||
from docreader import config
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser import Parser
|
||||
from docreader.proto import docreader_pb2_grpc
|
||||
from docreader.parser.registry import registry
|
||||
from docreader.proto.docreader_pb2 import (
|
||||
Chunk,
|
||||
Image,
|
||||
ReadConfig,
|
||||
ReadFromFileRequest,
|
||||
ReadFromURLRequest,
|
||||
ReadRequest,
|
||||
ReadResponse,
|
||||
StorageProvider,
|
||||
ImageRef,
|
||||
ListEnginesResponse,
|
||||
ParserEngineInfo,
|
||||
)
|
||||
from docreader.utils.request import init_logging_request_id, request_id_context
|
||||
|
||||
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
|
||||
# cannot be encoded to UTF-8
|
||||
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
|
||||
|
||||
|
||||
def to_valid_utf8_text(s: Optional[str]) -> str:
|
||||
"""Return a UTF-8 safe string for protobuf.
|
||||
|
||||
- Replace any surrogate code points with U+FFFD
|
||||
- Re-encode with errors='replace' to ensure valid UTF-8
|
||||
"""
|
||||
if not s:
|
||||
return ""
|
||||
s = _SURROGATE_RE.sub("\ufffd", s)
|
||||
return s.encode("utf-8", errors="replace").decode("utf-8")
|
||||
|
||||
|
||||
# Ensure no existing handlers
|
||||
for handler in logging.root.handlers[:]:
|
||||
logging.root.removeHandler(handler)
|
||||
|
||||
# Configure logging - use stdout
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
logging.root.addHandler(handler)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Initializing server logging")
|
||||
_level_name = (os.environ.get("LOG_LEVEL") or "INFO").upper()
|
||||
_level = getattr(logging, _level_name, logging.INFO)
|
||||
logging.root.setLevel(_level)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info("Initializing server logging, level=%s", _level_name)
|
||||
|
||||
# Initialize request ID logging
|
||||
init_logging_request_id()
|
||||
|
||||
|
||||
parser = Parser()
|
||||
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
|
||||
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
|
||||
|
||||
``images`` is a dict of {relative_path: raw_data} where raw_data is
|
||||
base64-encoded string or raw bytes.
|
||||
|
||||
def create_chunking_config(read_config: ReadConfig):
|
||||
"""Create ChunkingConfig from ReadConfig request.
|
||||
``storage_map`` is a dict from the request's config.image_storage map.
|
||||
When it contains a valid "provider" key, images are uploaded to shared object
|
||||
storage and storage_key is set (no inline bytes, saving gRPC message size).
|
||||
|
||||
Args:
|
||||
read_config: The read_config from the gRPC request
|
||||
|
||||
Returns:
|
||||
ChunkingConfig: Configured chunking configuration object
|
||||
Returns (image_dir_path, list[ImageRef]).
|
||||
"""
|
||||
# Extract chunking parameters
|
||||
chunk_size = read_config.chunk_size or 512
|
||||
chunk_overlap = read_config.chunk_overlap or 50
|
||||
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
|
||||
separators = (
|
||||
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
|
||||
)
|
||||
enable_multimodal = read_config.enable_multimodal or False
|
||||
import base64
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, "
|
||||
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
|
||||
)
|
||||
if not images:
|
||||
return "", []
|
||||
|
||||
# Extract storage config
|
||||
sc = read_config.storage_config
|
||||
storage_config = {
|
||||
"provider": StorageProvider.Name(sc.provider),
|
||||
"region": sc.region,
|
||||
"bucket_name": sc.bucket_name,
|
||||
"access_key_id": sc.access_key_id,
|
||||
"secret_access_key": sc.secret_access_key,
|
||||
"app_id": sc.app_id,
|
||||
"path_prefix": sc.path_prefix,
|
||||
}
|
||||
logger.info(
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, "
|
||||
f"bucket={storage_config['bucket_name']}"
|
||||
)
|
||||
|
||||
# Extract VLM config
|
||||
vlm_config = {
|
||||
"model_name": read_config.vlm_config.model_name,
|
||||
"base_url": read_config.vlm_config.base_url,
|
||||
"api_key": read_config.vlm_config.api_key,
|
||||
"interface_type": read_config.vlm_config.interface_type or "openai",
|
||||
mime_map = {
|
||||
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"Using VLM config: model={vlm_config['model_name']}, "
|
||||
f"base_url={vlm_config['base_url']}, "
|
||||
f"interface_type={vlm_config['interface_type']}"
|
||||
)
|
||||
storage_client = None
|
||||
use_storage = False
|
||||
provider = (storage_map or {}).get("provider", "")
|
||||
if provider:
|
||||
try:
|
||||
from docreader.parser.storage import create_storage
|
||||
storage_client = create_storage(storage_map)
|
||||
use_storage = True
|
||||
logger.info("Using shared storage (%s) for image upload", provider)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
|
||||
|
||||
# Create and return ChunkingConfig
|
||||
return ChunkingConfig(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separators=separators,
|
||||
enable_multimodal=enable_multimodal,
|
||||
storage_config=storage_config,
|
||||
vlm_config=vlm_config,
|
||||
)
|
||||
base_dir = CONFIG.image_output_dir
|
||||
output_dir = os.path.join(base_dir, request_id, "images")
|
||||
wrote_to_dir = False
|
||||
if not use_storage:
|
||||
try:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
wrote_to_dir = True
|
||||
except OSError:
|
||||
logger.warning("Cannot write to image output dir %s", output_dir)
|
||||
|
||||
refs = []
|
||||
for ref_path, b64data in images.items():
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64data)
|
||||
except Exception:
|
||||
img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data
|
||||
|
||||
fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png"
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
mime = mime_map.get(ext, "application/octet-stream")
|
||||
|
||||
if use_storage and storage_client:
|
||||
storage_key = storage_client.upload_bytes(img_bytes, ext)
|
||||
if storage_key:
|
||||
refs.append(ImageRef(
|
||||
filename=fname,
|
||||
original_ref=ref_path,
|
||||
mime_type=mime,
|
||||
storage_key=storage_key,
|
||||
))
|
||||
continue
|
||||
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
|
||||
|
||||
if wrote_to_dir:
|
||||
dest = os.path.join(output_dir, fname)
|
||||
try:
|
||||
with open(dest, "wb") as f:
|
||||
f.write(img_bytes)
|
||||
except OSError as e:
|
||||
logger.warning("Failed to write image %s: %s", dest, e)
|
||||
|
||||
refs.append(ImageRef(
|
||||
filename=fname,
|
||||
original_ref=ref_path,
|
||||
mime_type=mime,
|
||||
image_data=img_bytes,
|
||||
))
|
||||
|
||||
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
|
||||
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
|
||||
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
|
||||
return image_dir, refs
|
||||
|
||||
|
||||
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
|
|
@ -132,152 +144,90 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
|||
super().__init__()
|
||||
self.parser = Parser()
|
||||
|
||||
def ReadFromFile(self, request: ReadFromFileRequest, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
def Read(self, request: ReadRequest, context):
|
||||
"""Unified read: file mode (file_content set) or URL mode (url set)."""
|
||||
request_id = request.request_id or str(uuid.uuid4())
|
||||
is_url = bool(request.url)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
# Get file type
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
|
||||
)
|
||||
logger.info(f"File content size: {len(request.file_content)} bytes")
|
||||
cfg = request.config
|
||||
parser_engine = cfg.parser_engine if cfg else ""
|
||||
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
|
||||
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
|
||||
|
||||
# Create chunking config
|
||||
chunking_config = create_chunking_config(request.read_config)
|
||||
if is_url:
|
||||
logger.info("Read(URL): url=%s", request.url)
|
||||
result = self.parser.parse_url(
|
||||
request.url,
|
||||
request.title,
|
||||
parser_engine=parser_engine,
|
||||
engine_overrides=engine_overrides,
|
||||
)
|
||||
source_desc = request.url
|
||||
else:
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
"Read(File): file=%s, type=%s, size=%d bytes",
|
||||
request.file_name, file_type, len(request.file_content),
|
||||
)
|
||||
result = self.parser.parse_file(
|
||||
request.file_name,
|
||||
file_type,
|
||||
request.file_content,
|
||||
parser_engine=parser_engine,
|
||||
engine_overrides=engine_overrides,
|
||||
)
|
||||
source_desc = request.file_name
|
||||
|
||||
# Parse file
|
||||
logger.info("Starting file parsing process")
|
||||
result = self.parser.parse_file(
|
||||
request.file_name, file_type, request.file_content, chunking_config
|
||||
)
|
||||
|
||||
if not result:
|
||||
error_msg = "Failed to parse file"
|
||||
if not result or not result.content:
|
||||
error_msg = f"Failed to parse: {source_desc}"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse()
|
||||
|
||||
# Convert to protobuf message
|
||||
logger.info(
|
||||
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
# Build response, including image info
|
||||
response = ReadResponse(
|
||||
chunks=[
|
||||
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
|
||||
]
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading file: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def ReadFromURL(self, request: ReadFromURLRequest, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
logger.info(f"Received ReadFromURL request for URL: {request.url}")
|
||||
|
||||
# Create chunking config
|
||||
chunking_config = create_chunking_config(request.read_config)
|
||||
|
||||
# Parse URL
|
||||
logger.info("Starting URL parsing process")
|
||||
result = self.parser.parse_url(
|
||||
request.url, request.title, chunking_config
|
||||
)
|
||||
if not result:
|
||||
error_msg = "Failed to parse URL"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse(error=error_msg)
|
||||
|
||||
# Convert to protobuf message, including image info
|
||||
logger.info(
|
||||
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
|
||||
_c = to_valid_utf8_text
|
||||
image_dir, image_refs = _resolve_images(
|
||||
result.images, request_id, storage_map=storage_map
|
||||
)
|
||||
|
||||
response = ReadResponse(
|
||||
chunks=[
|
||||
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
|
||||
]
|
||||
markdown_content=_c(result.content),
|
||||
image_refs=image_refs,
|
||||
image_dir_path=image_dir,
|
||||
)
|
||||
logger.info(
|
||||
"Read response: content_len=%d, images=%d",
|
||||
len(result.content), len(image_refs),
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading URL: {str(e)}"
|
||||
error_msg = f"Error reading document: {e}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
logger.info("Traceback: %s", traceback.format_exc())
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def _convert_chunk_to_proto(self, chunk):
|
||||
"""Convert internal Chunk object to protobuf Chunk message
|
||||
Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates).
|
||||
"""
|
||||
# Clean helper for strings
|
||||
_c = to_valid_utf8_text
|
||||
|
||||
proto_chunk = Chunk(
|
||||
content=_c(getattr(chunk, "content", None)),
|
||||
seq=getattr(chunk, "seq", 0),
|
||||
start=getattr(chunk, "start", 0),
|
||||
end=getattr(chunk, "end", 0),
|
||||
)
|
||||
|
||||
# If chunk has images attribute and is not empty, add image info
|
||||
if hasattr(chunk, "images") and chunk.images:
|
||||
logger.info(
|
||||
f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}"
|
||||
def ListEngines(self, request, context):
|
||||
overrides = dict(getattr(request, "config_overrides", None) or {})
|
||||
engines_data = registry.list_engines(overrides=overrides or None)
|
||||
engines = [
|
||||
ParserEngineInfo(
|
||||
name=e["name"],
|
||||
description=e["description"],
|
||||
file_types=e["file_types"],
|
||||
available=e.get("available", True),
|
||||
unavailable_reason=e.get("unavailable_reason", ""),
|
||||
)
|
||||
for img_info in chunk.images:
|
||||
# img_info expected as dict
|
||||
proto_image = Image(
|
||||
url=_c(img_info.get("cos_url", "")),
|
||||
caption=_c(img_info.get("caption", "")),
|
||||
ocr_text=_c(img_info.get("ocr_text", "")),
|
||||
original_url=_c(img_info.get("original_url", "")),
|
||||
start=int(img_info.get("start", 0) or 0),
|
||||
end=int(img_info.get("end", 0) or 0),
|
||||
)
|
||||
proto_chunk.images.append(proto_image)
|
||||
|
||||
return proto_chunk
|
||||
for e in engines_data
|
||||
]
|
||||
return ListEnginesResponse(engines=engines)
|
||||
|
||||
|
||||
def main():
|
||||
# Print effective env/config at startup
|
||||
config.print_config()
|
||||
|
||||
# Create server
|
||||
server = grpc.server(
|
||||
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
|
||||
options=[
|
||||
|
|
@ -286,24 +236,18 @@ def main():
|
|||
],
|
||||
)
|
||||
|
||||
# Register services
|
||||
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
|
||||
|
||||
# Register health check service
|
||||
health_servicer = HealthServicer()
|
||||
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
|
||||
|
||||
# Set listen address
|
||||
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
|
||||
|
||||
# Start service
|
||||
server.start()
|
||||
|
||||
logger.info(f"Server started on port {CONFIG.grpc_port}")
|
||||
logger.info("Server started on port %d", CONFIG.grpc_port)
|
||||
logger.info("Server is ready to accept connections")
|
||||
|
||||
try:
|
||||
# Wait for service termination
|
||||
server.wait_for_termination()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Received termination signal, shutting down server")
|
||||
|
|
|
|||
|
|
@ -1,27 +1,17 @@
|
|||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkingConfig:
|
||||
"""
|
||||
Configuration for text chunking process.
|
||||
Controls how documents are split into smaller pieces for processing.
|
||||
"""Legacy config kept for backward compatibility.
|
||||
|
||||
After the lightweight refactoring, chunking is done in Go.
|
||||
This class is only kept so existing parser constructors don't break.
|
||||
"""
|
||||
|
||||
# Maximum size of each chunk in tokens/chars
|
||||
chunk_size: int = 512
|
||||
|
||||
# Number of tokens/chars to overlap between chunks
|
||||
chunk_overlap: int = 50
|
||||
|
||||
# Text separators in order of priority
|
||||
separators: list[str] = field(default_factory=lambda: ["\n\n", "\n", "。"])
|
||||
|
||||
# Whether to enable multimodal processing (text + images)
|
||||
separators: list[str] | None = None
|
||||
enable_multimodal: bool = False
|
||||
|
||||
# Preferred field name going forward
|
||||
storage_config: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# VLM configuration for image captioning
|
||||
vlm_config: dict[str, str] = field(default_factory=dict)
|
||||
storage_config: dict[str, str] | None = None
|
||||
vlm_config: dict[str, str] | None = None
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ The parsers extract content from documents and can split them into
|
|||
meaningful chunks for further processing and indexing.
|
||||
"""
|
||||
|
||||
from .csv_parser import CSVParser
|
||||
from .doc_parser import DocParser
|
||||
from .docx2_parser import Docx2Parser
|
||||
from .excel_parser import ExcelParser
|
||||
|
|
@ -21,19 +20,19 @@ from .image_parser import ImageParser
|
|||
from .markdown_parser import MarkdownParser
|
||||
from .parser import Parser
|
||||
from .pdf_parser import PDFParser
|
||||
from .text_parser import TextParser
|
||||
from .registry import ParserEngineRegistry, registry
|
||||
from .web_parser import WebParser
|
||||
|
||||
# Export public classes and modules
|
||||
__all__ = [
|
||||
"Docx2Parser", # Parser for .docx files (modern Word documents)
|
||||
"DocParser", # Parser for .doc files (legacy Word documents)
|
||||
"PDFParser", # Parser for PDF documents
|
||||
"MarkdownParser", # Parser for Markdown text files
|
||||
"TextParser", # Parser for plain text files
|
||||
"ImageParser", # Parser for images with text content
|
||||
"WebParser", # Parser for web pages
|
||||
"Parser", # Main parser factory that selects the appropriate parser
|
||||
"CSVParser", # Parser for CSV files
|
||||
"ExcelParser", # Parser for Excel files
|
||||
"Docx2Parser",
|
||||
"DocParser",
|
||||
"PDFParser",
|
||||
"MarkdownParser",
|
||||
"ImageParser",
|
||||
"WebParser",
|
||||
"Parser",
|
||||
"ExcelParser",
|
||||
"ParserEngineRegistry",
|
||||
"registry",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,387 +0,0 @@
|
|||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import ollama
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageUrl:
|
||||
"""Image URL data structure for caption requests."""
|
||||
|
||||
url: Optional[str] = None
|
||||
detail: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Content:
|
||||
"""Content data structure that can contain text or image URL."""
|
||||
|
||||
type: Optional[str] = None
|
||||
text: Optional[str] = None
|
||||
image_url: Optional[ImageUrl] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMessage:
|
||||
"""System message for VLM model requests."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserMessage:
|
||||
"""User message for VLM model requests, can contain multiple content items."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: List[Content] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompletionRequest:
|
||||
"""Request structure for VLM model completion API."""
|
||||
|
||||
model: str
|
||||
temperature: float
|
||||
top_p: float
|
||||
messages: List[Union[SystemMessage, UserMessage]]
|
||||
user: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
"""Model identifier structure."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelsResp:
|
||||
"""Response structure for available models API."""
|
||||
|
||||
data: List[Model] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
"""Message structure in API response."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
tool_calls: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Choice:
|
||||
"""Choice structure in API response."""
|
||||
|
||||
message: Optional[Message] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage information in API response."""
|
||||
|
||||
prompt_tokens: Optional[int] = 0
|
||||
total_tokens: Optional[int] = 0
|
||||
completion_tokens: Optional[int] = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptionChatResp:
|
||||
"""Response structure for caption chat API."""
|
||||
|
||||
id: Optional[str] = None
|
||||
created: Optional[int] = None
|
||||
model: Optional[Model] = None
|
||||
object: Optional[str] = None
|
||||
choices: List[Choice] = field(default_factory=list)
|
||||
usage: Optional[Usage] = None
|
||||
|
||||
@staticmethod
|
||||
def from_json(json_data: dict) -> "CaptionChatResp":
|
||||
"""
|
||||
Parse API response JSON into a CaptionChatResp object.
|
||||
|
||||
Args:
|
||||
json_data: The JSON response from the API
|
||||
|
||||
Returns:
|
||||
A parsed CaptionChatResp object
|
||||
"""
|
||||
logger.info("Parsing CaptionChatResp from JSON")
|
||||
# Manually parse nested fields with safe field extraction
|
||||
choices = []
|
||||
for choice in json_data.get("choices", []):
|
||||
message_data = choice.get("message", {})
|
||||
message = Message(
|
||||
role=message_data.get("role"),
|
||||
content=message_data.get("content"),
|
||||
tool_calls=message_data.get("tool_calls"),
|
||||
)
|
||||
choices.append(Choice(message=message))
|
||||
|
||||
# Handle usage with safe field extraction
|
||||
usage_data = json_data.get("usage", {})
|
||||
usage = None
|
||||
if usage_data:
|
||||
usage = Usage(
|
||||
prompt_tokens=usage_data.get("prompt_tokens", 0),
|
||||
total_tokens=usage_data.get("total_tokens", 0),
|
||||
completion_tokens=usage_data.get("completion_tokens", 0),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
|
||||
)
|
||||
return CaptionChatResp(
|
||||
id=json_data.get("id"),
|
||||
created=json_data.get("created"),
|
||||
model=json_data.get("model"),
|
||||
object=json_data.get("object"),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
def choice_data(self) -> str:
|
||||
"""
|
||||
Extract the content from the first choice in the response.
|
||||
|
||||
Returns:
|
||||
The content string from the first choice, or empty string if no choices
|
||||
"""
|
||||
if (
|
||||
not self.choices
|
||||
or not self.choices[0]
|
||||
or not self.choices[0].message
|
||||
or not self.choices[0].message.content
|
||||
):
|
||||
logger.warning("No choices available in response")
|
||||
return ""
|
||||
logger.info("Retrieving content from first choice")
|
||||
return self.choices[0].message.content
|
||||
|
||||
|
||||
class Caption:
|
||||
"""
|
||||
Service for generating captions for images using a Vision Language Model.
|
||||
Uses an external API to process images and return textual descriptions.
|
||||
"""
|
||||
|
||||
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
|
||||
"""
|
||||
Initialize the Caption service with configuration
|
||||
from parameters or environment variables.
|
||||
"""
|
||||
logger.info("Initializing Caption service")
|
||||
# Default prompt for image captioning in Chinese: "Briefly describe the main content of the image"
|
||||
self.prompt = """简单凝炼的描述图片的主要内容"""
|
||||
# API request timeout in seconds
|
||||
self.timeout = 30
|
||||
|
||||
# Use provided VLM config if available,
|
||||
# otherwise fall back to environment variables
|
||||
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
|
||||
# Build completion URL from provided base URL
|
||||
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
|
||||
self.model = vlm_config.get("model_name", "")
|
||||
self.api_key = vlm_config.get("api_key", "")
|
||||
# Interface type: "ollama" or "openai" (default)
|
||||
self.interface_type = vlm_config.get("interface_type", "openai").lower()
|
||||
else:
|
||||
# Fall back to environment variables if config not provided
|
||||
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
|
||||
self.model = CONFIG.vlm_model_name
|
||||
self.api_key = CONFIG.vlm_model_api_key
|
||||
self.interface_type = CONFIG.vlm_interface_type
|
||||
|
||||
# Validate interface type - must be either "ollama" or "openai"
|
||||
if self.interface_type not in ["ollama", "openai"]:
|
||||
logger.warning(
|
||||
f"Unknown interface type: {self.interface_type}, defaulting to openai"
|
||||
)
|
||||
self.interface_type = "openai"
|
||||
|
||||
logger.info(
|
||||
f"Configured with model: {self.model}, "
|
||||
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
|
||||
)
|
||||
|
||||
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
|
||||
"""
|
||||
Call the Caption API to generate a description for the given image.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
CaptionChatResp object if successful, None otherwise
|
||||
"""
|
||||
logger.info("Calling Caption API for image captioning")
|
||||
logger.info(f"Processing image data: {image_data[:50]}...")
|
||||
|
||||
# Route to appropriate API based on interface type
|
||||
if self.interface_type == "ollama":
|
||||
return self._call_ollama_api(image_data)
|
||||
else:
|
||||
return self._call_openai_api(image_data)
|
||||
|
||||
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call Ollama API for image captioning using base64 encoded image data."""
|
||||
|
||||
# Extract host URL by removing the chat completions endpoint
|
||||
# Handle both "/v1/chat/completions" and "/chat/completions" patterns
|
||||
host = self.completion_url.replace("/v1/chat/completions", "").replace("/chat/completions", "")
|
||||
|
||||
# Initialize Ollama client with host and timeout
|
||||
client = ollama.Client(
|
||||
host=host,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"Calling Ollama API with model: {self.model}")
|
||||
|
||||
# Call Ollama API with base64 encoded image
|
||||
# Prompt: "Briefly describe the main content of the image"
|
||||
response = client.generate(
|
||||
model=self.model,
|
||||
prompt="简单凝炼的描述图片的主要内容",
|
||||
images=[image_base64], # Pass base64 encoded image data
|
||||
options={
|
||||
"temperature": 0.1
|
||||
}, # Low temperature for more deterministic output
|
||||
stream=False,
|
||||
)
|
||||
|
||||
# Construct response object in standard format
|
||||
caption_resp = CaptionChatResp(
|
||||
id="ollama_response",
|
||||
created=int(time.time()),
|
||||
model=Model(id=self.model),
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(message=Message(role="assistant", content=response.response))
|
||||
],
|
||||
)
|
||||
|
||||
logger.info("Successfully received response from Ollama API")
|
||||
return caption_resp
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling Ollama API: {e}")
|
||||
return None
|
||||
|
||||
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call OpenAI-compatible API for image captioning."""
|
||||
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
|
||||
|
||||
# Construct user message with text prompt and base64 encoded image
|
||||
user_msg = UserMessage(
|
||||
role="user",
|
||||
content=[
|
||||
Content(type="text", text=self.prompt),
|
||||
Content(
|
||||
type="image_url",
|
||||
image_url=ImageUrl(
|
||||
url="data:image/png;base64," + image_base64, detail="auto"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Build completion request with model parameters
|
||||
gpt_req = CompletionRequest(
|
||||
model=self.model,
|
||||
temperature=0.3, # Moderate randomness for balanced output
|
||||
top_p=0.8, # Nucleus sampling parameter
|
||||
messages=[user_msg],
|
||||
user="abc",
|
||||
)
|
||||
|
||||
# Set up HTTP headers for the API request
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
# Add authorization header if API key is provided
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"Sending request to OpenAI-compatible API with model: {self.model}"
|
||||
)
|
||||
# Send POST request to the API endpoint
|
||||
response = requests.post(
|
||||
self.completion_url,
|
||||
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
# Check for successful response
|
||||
if response.status_code != 200:
|
||||
logger.error(
|
||||
f"OpenAI API returned non-200 status code: {response.status_code}"
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(f"Received from OpenAI with status: {response.status_code}")
|
||||
logger.info("Converting response to CaptionChatResp object")
|
||||
# Parse JSON response into structured object
|
||||
caption_resp = CaptionChatResp.from_json(response.json())
|
||||
|
||||
if caption_resp.usage:
|
||||
logger.info(
|
||||
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
|
||||
f"completion_tokens={caption_resp.usage.completion_tokens}"
|
||||
)
|
||||
|
||||
return caption_resp
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Request error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
|
||||
def get_caption(self, image_data: str) -> str:
|
||||
"""
|
||||
Get a caption for the provided image data.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
Caption text as string, or empty string if captioning failed
|
||||
"""
|
||||
logger.info("Getting caption for image")
|
||||
if not image_data or self.completion_url is None:
|
||||
logger.error("Image data is not set")
|
||||
return ""
|
||||
caption_resp = self._call_caption_api(image_data)
|
||||
if caption_resp:
|
||||
caption = caption_resp.choice_data()
|
||||
caption_length = len(caption)
|
||||
logger.info(f"Successfully generated caption of length {caption_length}")
|
||||
logger.info(
|
||||
f"Caption: {caption[:50]}..."
|
||||
if caption_length > 50
|
||||
else f"Caption: {caption}"
|
||||
)
|
||||
return caption
|
||||
logger.warning("Failed to get caption from Caption API")
|
||||
return ""
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
"""
|
||||
CSV Parser Module
|
||||
|
||||
This module provides a parser for CSV (Comma-Separated Values) files.
|
||||
It converts CSV data into a Document with structured chunks, where each row
|
||||
becomes a separate chunk with key-value pairs.
|
||||
"""
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from docreader.models.document import Chunk, Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CSVParser(BaseParser):
|
||||
"""
|
||||
Parser for CSV files that converts tabular data into structured text.
|
||||
|
||||
This parser reads CSV content and transforms each row into a formatted string
|
||||
with column-value pairs. Each row is stored as a separate Chunk in the Document,
|
||||
allowing for granular access to individual records.
|
||||
|
||||
The output format for each row is:
|
||||
"column1: value1, column2: value2, column3: value3\n"
|
||||
|
||||
Usage:
|
||||
parser = CSVParser()
|
||||
with open("data.csv", "rb") as f:
|
||||
document = parser.parse_into_text(f.read())
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse CSV content into a Document with structured chunks.
|
||||
|
||||
Each row in the CSV is converted into a formatted string and stored as
|
||||
a separate Chunk. The chunks maintain sequential order and track their
|
||||
position in the overall document.
|
||||
|
||||
Args:
|
||||
content: Raw bytes content of the CSV file
|
||||
|
||||
Returns:
|
||||
Document: A Document object containing:
|
||||
- content: Full text with all rows concatenated
|
||||
- chunks: List of Chunk objects, one per CSV row
|
||||
|
||||
Note:
|
||||
Bad lines in the CSV are automatically skipped using pandas'
|
||||
on_bad_lines="skip" parameter.
|
||||
"""
|
||||
chunks: List[Chunk] = []
|
||||
text: List[str] = []
|
||||
start, end = 0, 0
|
||||
|
||||
# Read CSV content into a pandas DataFrame, skipping malformed lines
|
||||
df = pd.read_csv(BytesIO(content), on_bad_lines="skip")
|
||||
|
||||
# Process each row in the DataFrame
|
||||
for i, (idx, row) in enumerate(df.iterrows()):
|
||||
# Format row as "column: value" pairs separated by commas
|
||||
content_row = (
|
||||
",".join(
|
||||
f"{col.strip()}: {str(row[col]).strip()}" for col in df.columns
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
# Update end position for this chunk
|
||||
end += len(content_row)
|
||||
text.append(content_row)
|
||||
|
||||
# Create a chunk for this row with position tracking
|
||||
chunks.append(Chunk(content=content_row, seq=i, start=start, end=end))
|
||||
# Update start position for next chunk
|
||||
start = end
|
||||
|
||||
return Document(
|
||||
content="".join(text),
|
||||
chunks=chunks,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage: Parse a CSV file and display its content
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
your_file = "/path/to/your/file.csv"
|
||||
parser = CSVParser()
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
document = parser.parse_into_text(content)
|
||||
# Display full document content
|
||||
logger.error(document.content)
|
||||
|
||||
# Display individual chunks (rows)
|
||||
for chunk in document.chunks:
|
||||
logger.error(chunk.content)
|
||||
|
|
@ -89,11 +89,33 @@ class DocxParser(BaseParser):
|
|||
logger.info(f"Setting max_workers to {max_workers} for document processing")
|
||||
|
||||
try:
|
||||
inline_images: Dict[str, str] = {}
|
||||
|
||||
def _inline_upload(local_path: str) -> str:
|
||||
"""Read temp image file, base64-encode, and return a ref path.
|
||||
|
||||
The Go-side ImageResolver (or main.py _resolve_images) handles
|
||||
actual storage upload from Document.images.
|
||||
"""
|
||||
import base64
|
||||
import uuid as _uuid
|
||||
|
||||
try:
|
||||
with open(local_path, "rb") as f:
|
||||
raw = f.read()
|
||||
ext = os.path.splitext(local_path)[1].lower() or ".png"
|
||||
ref = f"images/{_uuid.uuid4().hex}{ext}"
|
||||
inline_images[ref] = base64.b64encode(raw).decode()
|
||||
return ref
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read temp image %s: %s", local_path, exc)
|
||||
return ""
|
||||
|
||||
logger.info(f"Starting Docx processing with max_pages={self.max_pages}")
|
||||
docx_processor = Docx(
|
||||
max_image_size=self.max_image_size,
|
||||
enable_multimodal=self.enable_multimodal,
|
||||
upload_file=self.storage.upload_file,
|
||||
max_image_size=1920,
|
||||
enable_multimodal=True,
|
||||
upload_file=_inline_upload,
|
||||
)
|
||||
all_lines, tables = docx_processor(
|
||||
binary=content,
|
||||
|
|
@ -153,6 +175,7 @@ class DocxParser(BaseParser):
|
|||
f"generated {len(text)} characters of text"
|
||||
)
|
||||
|
||||
image_parts.update(inline_images)
|
||||
return DocumentModel(content=text, images=image_parts)
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing DOCX document: {str(e)}")
|
||||
|
|
|
|||
|
|
@ -5,44 +5,24 @@ import os
|
|||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
# Set up logger for this module
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageParser(BaseParser):
|
||||
"""
|
||||
Parser for image files with OCR capability.
|
||||
Extracts text from images and generates captions.
|
||||
"""Parser for standalone image files.
|
||||
|
||||
This parser handles image processing by:
|
||||
1. Uploading the image to storage
|
||||
2. Generating a descriptive caption
|
||||
3. Performing OCR to extract text content
|
||||
4. Returning a combined result with both text and image reference
|
||||
Returns the image as a markdown reference with the raw image data
|
||||
in Document.images so that the Go-side ImageResolver (or main.py's
|
||||
_resolve_images) can handle storage upload.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse image content into markdown text
|
||||
:param content: bytes content of the image
|
||||
:return: Document object
|
||||
"""
|
||||
logger.info(f"Parsing image content, size: {len(content)} bytes")
|
||||
logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
|
||||
|
||||
# Get file extension
|
||||
ext = os.path.splitext(self.file_name)[1].lower()
|
||||
ext = os.path.splitext(self.file_name)[1].lower() or ".png"
|
||||
ref_path = f"images/{self.file_name}"
|
||||
|
||||
# Upload image to storage
|
||||
image_url = self.storage.upload_bytes(content, file_ext=ext)
|
||||
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
|
||||
text = f""
|
||||
images = {ref_path: base64.b64encode(content).decode()}
|
||||
|
||||
if not image_url:
|
||||
logger.warning(f"Failed to upload image: {self.file_name}")
|
||||
return Document(content=f"{self.file_name}", images={})
|
||||
|
||||
# Generate markdown text
|
||||
text = f""
|
||||
images = {image_url: base64.b64encode(content).decode()}
|
||||
|
||||
# Create image object and add to map
|
||||
return Document(content=text, images=images)
|
||||
|
|
|
|||
|
|
@ -348,23 +348,11 @@ class MarkdownImageUtil:
|
|||
|
||||
|
||||
class MarkdownImageBase64(BaseParser):
|
||||
"""Parser for extracting and uploading base64 images from Markdown.
|
||||
"""Parser for extracting base64 images from Markdown.
|
||||
|
||||
This parser:
|
||||
1. Extracts base64-encoded images from Markdown content
|
||||
2. Uploads the decoded images to storage
|
||||
3. Replaces base64 data with uploaded URLs
|
||||
4. Returns a Document with clean Markdown and image mappings
|
||||
|
||||
Requires:
|
||||
- self.storage: Storage backend for uploading images
|
||||
|
||||
Example:
|
||||
>>> parser = MarkdownImageBase64(storage=my_storage)
|
||||
>>> content = b""
|
||||
>>> doc = parser.parse_into_text(content)
|
||||
>>> print(doc.content)
|
||||

|
||||
Extracts base64-encoded images, replaces them with path references,
|
||||
and returns the raw image data in Document.images for the Go-side
|
||||
ImageResolver (or main.py _resolve_images) to handle storage.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -372,41 +360,14 @@ class MarkdownImageBase64(BaseParser):
|
|||
self.image_helper = MarkdownImageUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse Markdown and process base64 images.
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
Returns:
|
||||
Document with:
|
||||
- content: Markdown with base64 replaced by URLs
|
||||
- images: Dict mapping URLs to base64 strings
|
||||
"""
|
||||
# Convert byte content to string using universal decoding method
|
||||
text = endecode.decode_bytes(content)
|
||||
# Extract base64 images and replace with temporary paths
|
||||
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
|
||||
|
||||
# Final image mapping: URL -> base64 string (for Document)
|
||||
images: Dict[str, str] = {}
|
||||
# Temporary mapping: temp_path -> uploaded_URL (for replacement)
|
||||
image_replace: Dict[str, str] = {}
|
||||
for ipath, raw_bytes in img_b64.items():
|
||||
images[ipath] = base64.b64encode(raw_bytes).decode()
|
||||
|
||||
logger.debug(f"Uploading {len(img_b64)} images from markdown")
|
||||
# Upload each extracted image to storage
|
||||
for ipath, b64_bytes in img_b64.items():
|
||||
# Get file extension for proper MIME type
|
||||
ext = os.path.splitext(ipath)[1].lower()
|
||||
# Upload binary data and get back URL
|
||||
image_url = self.storage.upload_bytes(b64_bytes, ext)
|
||||
|
||||
# Map temp path to uploaded URL for replacement
|
||||
image_replace[ipath] = image_url
|
||||
# Store base64 string in final images dict
|
||||
images[image_url] = base64.b64encode(b64_bytes).decode()
|
||||
|
||||
# Replace temporary paths with actual uploaded URLs
|
||||
text = self.image_helper.replace_path(text, image_replace)
|
||||
logger.debug("Extracted %d base64 images from markdown", len(images))
|
||||
return Document(content=text, images=images)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,329 +0,0 @@
|
|||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Optional
|
||||
|
||||
import markdownify
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.markdown_parser import MarkdownImageUtil, MarkdownTableFormatter
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StdMinerUParser(BaseParser):
|
||||
"""
|
||||
Standard MinerU Parser for document parsing.
|
||||
|
||||
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
|
||||
with support for tables, formulas, and images extraction.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
enable_markdownify: bool = True,
|
||||
mineru_endpoint: Optional[str] = None, # Added: 支持传入自定义 endpoint
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Initialize MinerU parser.
|
||||
|
||||
Args:
|
||||
enable_markdownify: Whether to convert HTML tables to markdown format
|
||||
mineru_endpoint: MinerU API endpoint URL
|
||||
**kwargs: Additional arguments passed to BaseParser
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
# Get MinerU endpoint from environment variable or parameter
|
||||
# Modified: 优先使用传入的参数,否则使用 Config
|
||||
base_url = mineru_endpoint if mineru_endpoint else CONFIG.mineru_endpoint
|
||||
self.minerU = base_url.rstrip("/") if base_url else ""
|
||||
|
||||
self.enable_markdownify = enable_markdownify
|
||||
# Helper for processing markdown images
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
# Pattern to match base64 encoded images
|
||||
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
|
||||
# Check if MinerU API is available
|
||||
self.enable = self.ping()
|
||||
|
||||
def ping(self, timeout: int = 5) -> bool:
|
||||
"""
|
||||
Check if MinerU API is available.
|
||||
|
||||
Args:
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
True if API is available, False otherwise
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
self.minerU + "/docs", timeout=timeout, allow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content into text using MinerU API.
|
||||
|
||||
Args:
|
||||
content: Raw document content in bytes
|
||||
|
||||
Returns:
|
||||
Document object containing parsed text and images
|
||||
"""
|
||||
if not self.enable:
|
||||
logger.debug("MinerU API is not enabled")
|
||||
return Document()
|
||||
|
||||
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
|
||||
md_content: str = ""
|
||||
images_b64: Dict[str, str] = {}
|
||||
try:
|
||||
# Call MinerU API to parse document
|
||||
response = requests.post(
|
||||
url=self.minerU + "/file_parse",
|
||||
data={
|
||||
"return_md": True, # Return markdown content
|
||||
"return_images": True, # Return extracted images
|
||||
"lang_list": ["ch", "en"], # Support Chinese and English
|
||||
"table_enable": True, # Enable table parsing
|
||||
"formula_enable": True, # Enable formula parsing
|
||||
"parse_method": "auto", # Auto detect parsing method
|
||||
"start_page_id": 0, # Start from first page
|
||||
"end_page_id": 99999, # Parse all pages
|
||||
"backend": "pipeline", # Use pipeline backend
|
||||
"response_format_zip": False, # Return JSON instead of ZIP
|
||||
"return_middle_json": False, # Don't return intermediate JSON
|
||||
"return_model_output": False, # Don't return model output
|
||||
"return_content_list": False, # Don't return content list
|
||||
},
|
||||
files={"files": content},
|
||||
timeout=1000,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()["results"]["files"]
|
||||
md_content = result["md_content"]
|
||||
images_b64 = result.get("images", {})
|
||||
except Exception as e:
|
||||
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
|
||||
return Document()
|
||||
|
||||
# Convert HTML tables in markdown to markdown table format
|
||||
if self.enable_markdownify:
|
||||
logger.debug("Converting HTML to Markdown")
|
||||
md_content = markdownify.markdownify(md_content)
|
||||
|
||||
images = {}
|
||||
image_replace = {}
|
||||
# Filter images that are actually used in markdown content
|
||||
# Some images in images_b64 may not be referenced in md_content
|
||||
# (e.g., images embedded in tables), so we need to filter them
|
||||
for ipath, b64_str in images_b64.items():
|
||||
# Skip images that are not referenced in markdown content
|
||||
if f"images/{ipath}" not in md_content:
|
||||
logger.debug(f"Image {ipath} not used in markdown")
|
||||
continue
|
||||
# Parse base64 image data
|
||||
match = self.base64_pattern.match(b64_str)
|
||||
if match:
|
||||
# Extract image format (e.g., png, jpg)
|
||||
file_ext = match.group(1)
|
||||
# Extract base64 encoded data
|
||||
b64_str = match.group(2)
|
||||
|
||||
# Decode base64 string to bytes
|
||||
image_bytes = endecode.encode_image(b64_str, errors="ignore")
|
||||
if not image_bytes:
|
||||
logger.error("Failed to decode base64 image skip it")
|
||||
continue
|
||||
|
||||
# Upload image to storage and get URL
|
||||
image_url = self.storage.upload_bytes(
|
||||
image_bytes, file_ext=f".{file_ext}"
|
||||
)
|
||||
|
||||
# Store image mapping for later use
|
||||
images[image_url] = b64_str
|
||||
# Prepare replacement mapping for markdown content
|
||||
image_replace[f"images/{ipath}"] = image_url
|
||||
|
||||
logger.info(f"Replaced {len(image_replace)} images in markdown")
|
||||
# Replace image paths in markdown with uploaded URLs
|
||||
text = self.image_helper.replace_path(md_content, image_replace)
|
||||
|
||||
logger.info(
|
||||
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
|
||||
)
|
||||
return Document(content=text, images=images)
|
||||
|
||||
|
||||
# Added: 新增 MinerUCloudParser 类,支持异步任务提交
|
||||
class MinerUCloudParser(StdMinerUParser):
|
||||
"""
|
||||
MinerU Parser for REMOTE/CLOUD API (Asynchronous).
|
||||
Uses the /submit -> /status -> /result workflow.
|
||||
"""
|
||||
|
||||
SUBMIT_TIMEOUT = 30
|
||||
POLL_INTERVAL = 2
|
||||
MAX_WAIT_TIME = 600
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content using Cloud MinerU API (Async/Polling).
|
||||
"""
|
||||
if not self.enable:
|
||||
return Document()
|
||||
|
||||
logger.info(f"Parsing PDF via Cloud MinerU API (size: {len(content)} bytes)")
|
||||
|
||||
try:
|
||||
# --- Step 1: Submit Task ---
|
||||
submit_url = f"{self.minerU}/submit"
|
||||
logger.info(f"Submitting task to {submit_url}")
|
||||
|
||||
response = requests.post(
|
||||
url=submit_url,
|
||||
files={"files": content},
|
||||
data={
|
||||
"enable_formula": "true",
|
||||
"enable_table": "true",
|
||||
"layout_model": "doclayout_yolo",
|
||||
"backend": "pipeline",
|
||||
},
|
||||
timeout=self.SUBMIT_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Robust task_id extraction
|
||||
resp_data = response.json()
|
||||
task_id = resp_data.get("task_id") or resp_data.get("data", {}).get("task_id")
|
||||
|
||||
if not task_id:
|
||||
raise ValueError(f"No task_id in response: {resp_data}")
|
||||
|
||||
logger.info(f"Task submitted, ID: {task_id}, waiting for completion...")
|
||||
|
||||
# --- Step 2: Poll Status ---
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
if time.time() - start_time > self.MAX_WAIT_TIME:
|
||||
raise TimeoutError(f"Task {task_id} timed out after {self.MAX_WAIT_TIME}s")
|
||||
|
||||
try:
|
||||
status_resp = requests.get(
|
||||
f"{self.minerU}/status/{task_id}",
|
||||
timeout=10
|
||||
)
|
||||
status_resp.raise_for_status()
|
||||
status_data = status_resp.json()
|
||||
except requests.RequestException as e:
|
||||
logger.warning(f"Status check failed for {task_id}: {e}. Retrying...")
|
||||
time.sleep(self.POLL_INTERVAL)
|
||||
continue
|
||||
|
||||
state = status_data.get("status") or status_data.get("state")
|
||||
|
||||
if state in ["done", "success"]:
|
||||
break
|
||||
elif state == "failed":
|
||||
error_msg = status_data.get("error") or "Unknown error"
|
||||
raise RuntimeError(f"Task {task_id} failed: {error_msg}")
|
||||
else:
|
||||
time.sleep(self.POLL_INTERVAL)
|
||||
|
||||
# --- Step 3: Get Result ---
|
||||
result_resp = requests.get(
|
||||
f"{self.minerU}/result/{task_id}",
|
||||
timeout=30
|
||||
)
|
||||
result_resp.raise_for_status()
|
||||
result_json = result_resp.json()
|
||||
|
||||
# Normalize result data
|
||||
result_data = result_json.get("result", result_json)
|
||||
|
||||
md_content = result_data.get("md_content", "")
|
||||
images_b64 = result_data.get("images", {})
|
||||
|
||||
# 使用父类的方法处理图片和Markdown转换 (复用现有逻辑)
|
||||
|
||||
# Convert HTML tables
|
||||
if self.enable_markdownify:
|
||||
md_content = markdownify.markdownify(md_content)
|
||||
|
||||
images = {}
|
||||
image_replace = {}
|
||||
|
||||
for ipath, b64_str in images_b64.items():
|
||||
if f"images/{ipath}" not in md_content:
|
||||
continue
|
||||
match = self.base64_pattern.match(b64_str)
|
||||
if match:
|
||||
file_ext = match.group(1)
|
||||
b64_str_clean = match.group(2)
|
||||
image_bytes = endecode.encode_image(b64_str_clean, errors="ignore")
|
||||
if not image_bytes: continue
|
||||
|
||||
if self.storage:
|
||||
image_url = self.storage.upload_bytes(image_bytes, file_ext=f".{file_ext}")
|
||||
images[image_url] = b64_str_clean
|
||||
image_replace[f"images/{ipath}"] = image_url
|
||||
|
||||
if image_replace:
|
||||
md_content = self.image_helper.replace_path(md_content, image_replace)
|
||||
|
||||
return Document(content=md_content, images=images)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Cloud MinerU parsing failed: {e}", exc_info=True)
|
||||
return Document()
|
||||
|
||||
|
||||
class MinerUParser(PipelineParser):
|
||||
"""
|
||||
MinerU Parser with pipeline processing.
|
||||
|
||||
This parser combines StdMinerUParser for document parsing and
|
||||
MarkdownTableFormatter for table formatting in a pipeline.
|
||||
"""
|
||||
|
||||
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
|
||||
# Example usage for testing
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
# Configure your file path and MinerU endpoint
|
||||
your_file = "/path/to/your/file.pdf"
|
||||
|
||||
# Added: 修改为 Localhost 方便测试
|
||||
test_endpoint = "http://localhost:9987"
|
||||
os.environ["MINERU_ENDPOINT"] = test_endpoint
|
||||
|
||||
# Create parser instance
|
||||
# Modified: 传入 endpoint
|
||||
parser = MinerUParser(mineru_endpoint=test_endpoint)
|
||||
|
||||
# Parse PDF file
|
||||
if os.path.exists(your_file):
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
document = parser.parse_into_text(content)
|
||||
logger.error(document.content)
|
||||
else:
|
||||
print(f"File not found: {your_file}")
|
||||
|
|
@ -1,178 +1,82 @@
|
|||
import logging
|
||||
from typing import Dict, Type
|
||||
from typing import Any, Optional
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.csv_parser import CSVParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
from docreader.parser.text_parser import TextParser
|
||||
from docreader.parser.registry import registry
|
||||
from docreader.parser.web_parser import WebParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Document parser facade that integrates all specialized parsers.
|
||||
Provides a unified interface for parsing various document types.
|
||||
"""Document parser facade (lightweight version).
|
||||
|
||||
Converts files/URLs to markdown + image references.
|
||||
No chunking, no storage, no OCR, no VLM.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Initialize all parser types - maps file extensions to their corresponding parser classes
|
||||
self.parsers: Dict[str, Type[BaseParser]] = {
|
||||
# Document formats
|
||||
"docx": Docx2Parser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"txt": TextParser,
|
||||
# Image formats - all use the same ImageParser
|
||||
"jpg": ImageParser,
|
||||
"jpeg": ImageParser,
|
||||
"png": ImageParser,
|
||||
"gif": ImageParser,
|
||||
"bmp": ImageParser,
|
||||
"tiff": ImageParser,
|
||||
"webp": ImageParser,
|
||||
# Alternative markdown extension
|
||||
"markdown": MarkdownParser,
|
||||
# Spreadsheet formats
|
||||
"csv": CSVParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
}
|
||||
self.registry = registry
|
||||
logger.info(
|
||||
"Parser initialized with %d parsers: %s",
|
||||
len(self.parsers),
|
||||
", ".join(self.parsers.keys()),
|
||||
"Parser initialized with engines: %s",
|
||||
", ".join(self.registry.get_engine_names()),
|
||||
)
|
||||
|
||||
def get_parser(self, file_type: str) -> Type[BaseParser]:
|
||||
"""
|
||||
Get parser class for the specified file type.
|
||||
|
||||
Args:
|
||||
file_type: The file extension or type identifier
|
||||
|
||||
Returns:
|
||||
Parser class for the file type, or None if unsupported
|
||||
"""
|
||||
# Look up parser by file type (case-insensitive)
|
||||
parser = self.parsers.get(file_type.lower())
|
||||
if not parser:
|
||||
# Raise error if file type is not supported
|
||||
raise ValueError(f"Unsupported file type: {file_type}")
|
||||
return parser
|
||||
|
||||
def parse_file(
|
||||
self,
|
||||
file_name: str,
|
||||
file_type: str,
|
||||
content: bytes,
|
||||
config: ChunkingConfig,
|
||||
parser_engine: Optional[str] = None,
|
||||
engine_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> Document:
|
||||
"""
|
||||
Parse file content using appropriate parser based on file type.
|
||||
|
||||
Args:
|
||||
file_name: Name of the file being parsed
|
||||
file_type: Type/extension of the file
|
||||
content: Raw file content as bytes
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing file: {file_name} with type: {file_type}")
|
||||
"""Parse file content to markdown."""
|
||||
engine = parser_engine or ""
|
||||
overrides = engine_overrides or {}
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, "
|
||||
f"overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
"Parsing file: %s, type: %s, engine: %s",
|
||||
file_name,
|
||||
file_type,
|
||||
engine or "builtin",
|
||||
)
|
||||
|
||||
# Get appropriate parser class for the file type
|
||||
cls = self.get_parser(file_type)
|
||||
|
||||
# Create parser instance with configuration
|
||||
logger.info(f"Creating parser instance for {file_type} file")
|
||||
cls = self.registry.get_parser_class(engine, file_type)
|
||||
logger.info(
|
||||
"Creating %s parser instance for %s file",
|
||||
cls.__name__,
|
||||
file_type,
|
||||
)
|
||||
parser = cls(
|
||||
file_name=file_name,
|
||||
file_type=file_type,
|
||||
chunk_size=config.chunk_size, # Size of each text chunk
|
||||
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config for advanced options
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
**overrides,
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
# Execute the parsing process
|
||||
logger.info("Starting to parse file content, size: %d bytes", len(content))
|
||||
result = parser.parse(content)
|
||||
|
||||
# Validate parsing results and log warnings if needed
|
||||
if not result.content:
|
||||
logger.warning(f"Parser returned empty content for file: {file_name}")
|
||||
elif not result.chunks:
|
||||
logger.warning(f"Parser returned empty chunks for file: {file_name}")
|
||||
elif result.chunks[0]:
|
||||
# Log first chunk size for debugging
|
||||
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
|
||||
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
|
||||
logger.warning("Parser returned empty content for file: %s", file_name)
|
||||
logger.info(
|
||||
"Parsed file %s, content length=%d", file_name, len(result.content)
|
||||
)
|
||||
return result
|
||||
|
||||
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
|
||||
"""
|
||||
Parse content from a URL using the WebParser.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
title: Title of the webpage (for metadata)
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing URL: {url}, title: {title}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, "
|
||||
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
# Create web parser instance with configuration
|
||||
logger.info("Creating WebParser instance")
|
||||
parser = WebParser(
|
||||
title=title, # Webpage title for metadata
|
||||
chunk_size=config.chunk_size, # Size of each text chunk
|
||||
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
)
|
||||
def parse_url(
|
||||
self,
|
||||
url: str,
|
||||
title: str,
|
||||
parser_engine: Optional[str] = None,
|
||||
engine_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> Document:
|
||||
"""Parse content from a URL to markdown."""
|
||||
logger.info("Parsing URL: %s, title: %s", url, title)
|
||||
|
||||
parser = WebParser(title=title)
|
||||
logger.info("Starting to parse URL content")
|
||||
# Parse URL content (encode URL string to bytes as required by parser interface)
|
||||
result = parser.parse(url.encode())
|
||||
|
||||
# Validate parsing results and log warnings if needed
|
||||
if not result.content:
|
||||
logger.warning(f"Parser returned empty content for url: {url}")
|
||||
elif not result.chunks:
|
||||
logger.warning(f"Parser returned empty chunks for url: {url}")
|
||||
elif result.chunks[0]:
|
||||
# Log first chunk size for debugging
|
||||
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
|
||||
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
|
||||
logger.warning("Parser returned empty content for url: %s", url)
|
||||
logger.info("Parsed url %s, content length=%d", url, len(result.content))
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from docreader.parser.chain_parser import FirstParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.mineru_parser import MinerUParser
|
||||
|
||||
|
||||
class PDFParser(FirstParser):
|
||||
|
|
@ -13,4 +12,4 @@ class PDFParser(FirstParser):
|
|||
The first successful parser result will be returned.
|
||||
"""
|
||||
# Parser classes to try in order (chain of responsibility pattern)
|
||||
_parser_cls = (MinerUParser, MarkitdownParser)
|
||||
_parser_cls = (MarkitdownParser,)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,160 @@
|
|||
import logging
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Type
|
||||
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BUILTIN_ENGINE = "builtin"
|
||||
|
||||
|
||||
class ParserEngineRegistry:
|
||||
"""Registry for parser engines.
|
||||
|
||||
Each engine maps file extensions to parser classes.
|
||||
When a requested engine doesn't support a file type, the registry
|
||||
falls back to the builtin engine automatically.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
|
||||
self._descriptions: Dict[str, str] = {}
|
||||
self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
|
||||
self._unavailable_hint: Dict[str, str] = {}
|
||||
|
||||
def register(
|
||||
self,
|
||||
name: str,
|
||||
file_types: Dict[str, Type[BaseParser]],
|
||||
description: str = "",
|
||||
check_available: Callable[..., Tuple[bool, str]] | None = None,
|
||||
unavailable_hint: str = "",
|
||||
):
|
||||
self._engines[name] = file_types
|
||||
self._descriptions[name] = description
|
||||
if check_available is not None:
|
||||
self._check_available[name] = check_available
|
||||
self._unavailable_hint[name] = unavailable_hint
|
||||
logger.info(
|
||||
"Registered parser engine '%s' with file types: %s",
|
||||
name,
|
||||
", ".join(file_types.keys()),
|
||||
)
|
||||
|
||||
def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
|
||||
"""Resolve parser class for the given engine and file type.
|
||||
|
||||
Falls back to builtin engine when the requested engine doesn't
|
||||
support the file type.
|
||||
"""
|
||||
ft = file_type.lower()
|
||||
|
||||
if engine and engine in self._engines:
|
||||
cls = self._engines[engine].get(ft)
|
||||
if cls:
|
||||
logger.info("Using engine '%s' for file type '%s'", engine, ft)
|
||||
return cls
|
||||
logger.info(
|
||||
"Engine '%s' does not support '%s', falling back to builtin",
|
||||
engine,
|
||||
ft,
|
||||
)
|
||||
|
||||
builtin = self._engines.get(BUILTIN_ENGINE, {})
|
||||
cls = builtin.get(ft)
|
||||
if cls:
|
||||
return cls
|
||||
|
||||
raise ValueError(f"Unsupported file type: {file_type}")
|
||||
|
||||
def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
|
||||
"""Return metadata for all registered engines, including availability.
|
||||
|
||||
Args:
|
||||
overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
|
||||
forwarded to each engine's check_available function.
|
||||
"""
|
||||
result = []
|
||||
for name, parsers in self._engines.items():
|
||||
available = True
|
||||
unavailable_reason = ""
|
||||
check = self._check_available.get(name)
|
||||
if check is not None:
|
||||
try:
|
||||
available, unavailable_reason = check(overrides)
|
||||
except Exception as e:
|
||||
available = False
|
||||
unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
|
||||
if not available and not unavailable_reason:
|
||||
unavailable_reason = self._unavailable_hint.get(name, "不可用")
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": self._descriptions.get(name, ""),
|
||||
"file_types": sorted(parsers.keys()),
|
||||
"available": available,
|
||||
"unavailable_reason": unavailable_reason,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def get_engine_names(self) -> List[str]:
|
||||
return list(self._engines.keys())
|
||||
|
||||
|
||||
def _build_default_registry() -> ParserEngineRegistry:
|
||||
"""Create and populate the default registry with all known engines."""
|
||||
reg = ParserEngineRegistry()
|
||||
|
||||
_image_types = {
|
||||
ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
|
||||
}
|
||||
|
||||
reg.register(
|
||||
BUILTIN_ENGINE,
|
||||
{
|
||||
"docx": Docx2Parser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"markdown": MarkdownParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
**_image_types,
|
||||
},
|
||||
description="内置解析引擎",
|
||||
)
|
||||
|
||||
reg.register(
|
||||
"markitdown",
|
||||
{
|
||||
"md": MarkitdownParser,
|
||||
"markdown": MarkitdownParser,
|
||||
"pdf": MarkitdownParser,
|
||||
"docx": MarkitdownParser,
|
||||
"doc": MarkitdownParser,
|
||||
"pptx": MarkitdownParser,
|
||||
"ppt": MarkitdownParser,
|
||||
"xlsx": MarkitdownParser,
|
||||
"xls": MarkitdownParser,
|
||||
"csv": MarkitdownParser,
|
||||
},
|
||||
description="MarkItDown 解析引擎(微软 MarkItDown 库)",
|
||||
)
|
||||
|
||||
# NOTE: Engine listing is managed by Go-side engine registry
|
||||
# (docparser.ListAllEngines). The Python list_engines method is kept for
|
||||
# backward compatibility with the gRPC ListEngines RPC but the Go app
|
||||
# no longer calls it. MinerU engines are handled natively by Go.
|
||||
|
||||
return reg
|
||||
|
||||
|
||||
registry = _build_default_registry()
|
||||
|
|
@ -5,96 +5,70 @@ import os
|
|||
import traceback
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict
|
||||
from typing import Dict, Optional
|
||||
|
||||
from minio import Minio
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cfg(storage_config: Optional[Dict], key: str, *env_keys: str, default: str = "") -> str:
|
||||
"""Read a value from storage_config dict, falling back to env vars."""
|
||||
if storage_config:
|
||||
v = storage_config.get(key, "")
|
||||
if v:
|
||||
return str(v)
|
||||
for ek in env_keys:
|
||||
v = os.environ.get(ek, "")
|
||||
if v:
|
||||
return v
|
||||
return default
|
||||
|
||||
|
||||
class Storage(ABC):
|
||||
"""Abstract base class for object storage operations"""
|
||||
|
||||
@abstractmethod
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to object storage
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to object storage
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CosStorage(Storage):
|
||||
"""Tencent Cloud COS storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize COS storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.region, self.prefix = (
|
||||
self._init_cos_client()
|
||||
)
|
||||
|
||||
def _init_cos_client(self):
|
||||
"""Initialize Tencent Cloud COS client"""
|
||||
try:
|
||||
# Use provided COS config if available,
|
||||
# otherwise fall back to environment variables
|
||||
if self.storage_config and self.storage_config.get("access_key_id") != "":
|
||||
cos_config = self.storage_config
|
||||
secret_id = cos_config.get("access_key_id")
|
||||
secret_key = cos_config.get("secret_access_key")
|
||||
region = cos_config.get("region")
|
||||
bucket_name = cos_config.get("bucket_name")
|
||||
appid = cos_config.get("app_id")
|
||||
prefix = cos_config.get("path_prefix", "")
|
||||
else:
|
||||
# Get COS configuration from environment variables
|
||||
secret_id = CONFIG.cos_secret_id
|
||||
secret_key = CONFIG.cos_secret_key
|
||||
region = CONFIG.cos_region
|
||||
bucket_name = CONFIG.cos_bucket_name
|
||||
appid = CONFIG.cos_app_id
|
||||
prefix = CONFIG.cos_path_prefix
|
||||
|
||||
enable_old_domain = CONFIG.cos_enable_old_domain
|
||||
sc = self.storage_config
|
||||
secret_id = _cfg(sc, "access_key_id", "COS_SECRET_ID")
|
||||
secret_key = _cfg(sc, "secret_access_key", "COS_SECRET_KEY")
|
||||
region = _cfg(sc, "region", "COS_REGION")
|
||||
bucket_name = _cfg(sc, "bucket_name", "COS_BUCKET_NAME")
|
||||
appid = _cfg(sc, "app_id", "COS_APP_ID")
|
||||
prefix = _cfg(sc, "path_prefix", "COS_PATH_PREFIX")
|
||||
enable_old_domain = os.environ.get("COS_ENABLE_OLD_DOMAIN", "").lower() in ("1", "true", "yes")
|
||||
|
||||
if not all([secret_id, secret_key, region, bucket_name, appid]):
|
||||
logger.error(
|
||||
"Incomplete COS configuration, missing environment variables"
|
||||
f"secret_id: {secret_id}, secret_key: {secret_key}, "
|
||||
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
|
||||
"Incomplete COS configuration: "
|
||||
"secret_id=%s, region=%s, bucket=%s, appid=%s",
|
||||
bool(secret_id), region, bucket_name, appid,
|
||||
)
|
||||
return None, None, None, None
|
||||
|
||||
# Initialize COS configuration
|
||||
logger.info(
|
||||
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
|
||||
)
|
||||
logger.info("Initializing COS client: region=%s, bucket=%s", region, bucket_name)
|
||||
config = CosConfig(
|
||||
Appid=appid,
|
||||
Region=region,
|
||||
|
|
@ -102,93 +76,50 @@ class CosStorage(Storage):
|
|||
SecretKey=secret_key,
|
||||
EnableOldDomain=enable_old_domain,
|
||||
)
|
||||
|
||||
# Create client
|
||||
client = CosS3Client(config)
|
||||
return client, bucket_name, region, prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize COS client: {str(e)}")
|
||||
logger.error("Failed to initialize COS client: %s", e)
|
||||
return None, None, None, None
|
||||
|
||||
def _get_download_url(self, bucket_name, region, object_key):
|
||||
"""Generate COS object URL
|
||||
|
||||
Args:
|
||||
bucket_name: Bucket name
|
||||
region: Region
|
||||
object_key: Object key
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to COS: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to COS")
|
||||
self.client.upload_file(
|
||||
Bucket=self.bucket_name,
|
||||
LocalFilePath=file_path,
|
||||
Key=object_key,
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to COS: {file_url}")
|
||||
logger.info("COS upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to COS: {str(e)}")
|
||||
logger.error("COS upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = (
|
||||
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket_name, Body=content, Key=object_key
|
||||
)
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
|
||||
logger.info("COS upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to COS: {str(e)}")
|
||||
logger.error("COS upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
|
@ -196,67 +127,34 @@ class CosStorage(Storage):
|
|||
class MinioStorage(Storage):
|
||||
"""MinIO storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize MinIO storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
|
||||
self._init_minio_client()
|
||||
)
|
||||
|
||||
def _init_minio_client(self):
|
||||
"""Initialize MinIO client from environment variables or injected config.
|
||||
|
||||
If storage_config contains valid configuration, prefer those values
|
||||
to override environment variables.
|
||||
"""
|
||||
try:
|
||||
# Get configuration from storage_config with environment variables as fallback
|
||||
# Each field can independently fall back to environment variables
|
||||
access_key = (
|
||||
self.storage_config.get("access_key_id")
|
||||
if self.storage_config and self.storage_config.get("access_key_id")
|
||||
else CONFIG.minio_access_key_id
|
||||
)
|
||||
secret_key = (
|
||||
self.storage_config.get("secret_access_key")
|
||||
if self.storage_config and self.storage_config.get("secret_access_key")
|
||||
else CONFIG.minio_secret_access_key
|
||||
)
|
||||
bucket_name = (
|
||||
self.storage_config.get("bucket_name")
|
||||
if self.storage_config and self.storage_config.get("bucket_name")
|
||||
else CONFIG.minio_bucket_name
|
||||
)
|
||||
path_prefix_raw = (
|
||||
self.storage_config.get("path_prefix")
|
||||
if self.storage_config and self.storage_config.get("path_prefix")
|
||||
else CONFIG.minio_path_prefix
|
||||
)
|
||||
sc = self.storage_config
|
||||
access_key = _cfg(sc, "access_key_id", "MINIO_ACCESS_KEY_ID")
|
||||
secret_key = _cfg(sc, "secret_access_key", "MINIO_SECRET_ACCESS_KEY")
|
||||
bucket_name = _cfg(sc, "bucket_name", "MINIO_BUCKET_NAME")
|
||||
path_prefix_raw = _cfg(sc, "path_prefix", "MINIO_PATH_PREFIX")
|
||||
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
|
||||
|
||||
endpoint = CONFIG.minio_endpoint
|
||||
use_ssl = CONFIG.minio_use_ssl
|
||||
endpoint = _cfg(sc, "endpoint", "MINIO_ENDPOINT")
|
||||
use_ssl = os.environ.get("MINIO_USE_SSL", "").lower() in ("1", "true", "yes")
|
||||
|
||||
if not all([endpoint, access_key, secret_key, bucket_name]):
|
||||
logger.error(
|
||||
"Incomplete MinIO configuration, missing environment variables"
|
||||
)
|
||||
logger.error("Incomplete MinIO configuration")
|
||||
return None, None, None, None, None
|
||||
|
||||
# Initialize client
|
||||
client = Minio(
|
||||
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
|
||||
)
|
||||
|
||||
# Ensure bucket exists
|
||||
found = client.bucket_exists(bucket_name)
|
||||
if not found:
|
||||
client.make_bucket(bucket_name)
|
||||
# Set public read policy for the bucket
|
||||
policy = (
|
||||
"{"
|
||||
'"Version":"2012-10-17",'
|
||||
|
|
@ -273,51 +171,26 @@ class MinioStorage(Storage):
|
|||
|
||||
return client, bucket_name, use_ssl, endpoint, path_prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize MinIO client: {str(e)}")
|
||||
logger.error("Failed to initialize MinIO client: %s", e)
|
||||
return None, None, None, None, None
|
||||
|
||||
def _get_download_url(self, object_key: str):
|
||||
"""Construct a public URL for MinIO object.
|
||||
|
||||
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
|
||||
"""
|
||||
# 1. Use public endpoint if provided
|
||||
endpoint = CONFIG.minio_public_endpoint
|
||||
if endpoint:
|
||||
return f"{endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
# 2. Use SSL if enabled
|
||||
if self.use_ssl:
|
||||
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
# 3. Use HTTP default
|
||||
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
public_endpoint = os.environ.get("MINIO_PUBLIC_ENDPOINT", "")
|
||||
if public_endpoint:
|
||||
return f"{public_endpoint}/{self.bucket_name}/{object_key}"
|
||||
scheme = "https" if self.use_ssl else "http"
|
||||
return f"{scheme}://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to MinIO
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to MinIO: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_name = os.path.basename(file_path)
|
||||
object_key = (
|
||||
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
if self.path_prefix
|
||||
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
)
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to MinIO")
|
||||
with open(file_path, "rb") as file_data:
|
||||
file_size = os.path.getsize(file_path)
|
||||
self.client.put_object(
|
||||
|
|
@ -327,38 +200,22 @@ class MinioStorage(Storage):
|
|||
length=file_size,
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
|
||||
logger.info("MinIO upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to MinIO: {str(e)}")
|
||||
logger.error("MinIO upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to MinIO
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = (
|
||||
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.path_prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
self.client.put_object(
|
||||
self.bucket_name or "",
|
||||
object_key,
|
||||
|
|
@ -367,52 +224,67 @@ class MinioStorage(Storage):
|
|||
content_type="application/octet-stream",
|
||||
)
|
||||
file_url = self._get_download_url(object_key)
|
||||
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
|
||||
logger.info("MinIO upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
|
||||
logger.error("MinIO upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
||||
class LocalStorage(Storage):
|
||||
"""Local file system storage implementation"""
|
||||
"""Local file system storage implementation.
|
||||
|
||||
def __init__(self, storage_config: Dict[str, str] = {}):
|
||||
self.storage_config = storage_config
|
||||
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
|
||||
self.image_dir = os.path.join(base_dir, "images")
|
||||
Saves files under base_dir and returns web-accessible URL paths
|
||||
(e.g. /files/images/uuid.jpg) so that the Go app can serve them.
|
||||
"""
|
||||
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
sc = storage_config or {}
|
||||
self.base_dir = (
|
||||
sc.get("base_dir")
|
||||
or os.environ.get("LOCAL_STORAGE_BASE_DIR", "/data/files")
|
||||
)
|
||||
path_prefix = (sc.get("path_prefix") or "").strip().strip("/")
|
||||
if path_prefix:
|
||||
self.image_dir = os.path.join(self.base_dir, path_prefix, "images")
|
||||
else:
|
||||
self.image_dir = os.path.join(self.base_dir, "images")
|
||||
self.url_prefix = (
|
||||
sc.get("url_prefix")
|
||||
or os.environ.get("LOCAL_STORAGE_URL_PREFIX", "/files")
|
||||
)
|
||||
os.makedirs(self.image_dir, exist_ok=True)
|
||||
|
||||
def _to_url(self, fpath: str) -> str:
|
||||
if self.url_prefix:
|
||||
rel = os.path.relpath(fpath, self.base_dir)
|
||||
return f"{self.url_prefix}/{rel}"
|
||||
return fpath
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
logger.info(f"Uploading file to local storage: {file_path}")
|
||||
return file_path
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
logger.info(f"Uploading file to local storage: {len(content)} bytes")
|
||||
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
|
||||
with open(fname, "wb") as f:
|
||||
fpath = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
|
||||
with open(fpath, "wb") as f:
|
||||
f.write(content)
|
||||
return fname
|
||||
url = self._to_url(fpath)
|
||||
logger.info("Local storage saved: %s -> %s", fpath, url)
|
||||
return url
|
||||
|
||||
|
||||
class Base64Storage(Storage):
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
logger.info(f"Uploading file to base64 storage: {file_path}")
|
||||
return file_path
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
|
||||
file_ext = file_ext.lstrip(".")
|
||||
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
|
||||
|
||||
|
||||
class DummyStorage(Storage):
|
||||
"""Dummy storage implementation.
|
||||
|
||||
It is used in tests or in environments where object storage is disabled.
|
||||
All upload methods return empty string.
|
||||
"""
|
||||
"""Dummy storage — all uploads return empty string."""
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
return ""
|
||||
|
|
@ -421,26 +293,30 @@ class DummyStorage(Storage):
|
|||
return ""
|
||||
|
||||
|
||||
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
|
||||
"""Create a storage instance based on configuration or environment variables
|
||||
def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
|
||||
"""Create a storage instance based on storage_config dict.
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration dictionary
|
||||
|
||||
Returns:
|
||||
Storage instance
|
||||
The ``provider`` key in storage_config determines the backend:
|
||||
minio, cos, local, base64.
|
||||
Falls back to STORAGE_TYPE env var, then ``local``.
|
||||
"""
|
||||
storage_type = CONFIG.storage_type
|
||||
storage_type = ""
|
||||
if storage_config:
|
||||
storage_type = str(storage_config.get("provider", storage_type)).lower()
|
||||
logger.info(f"Creating {storage_type} storage instance")
|
||||
provider = str(storage_config.get("provider", "")).lower().strip()
|
||||
if provider and provider not in ("unspecified", "storage_provider_unspecified"):
|
||||
storage_type = provider
|
||||
|
||||
if not storage_type:
|
||||
storage_type = os.environ.get("STORAGE_TYPE", "local").lower().strip()
|
||||
|
||||
logger.info("Creating %s storage instance", storage_type)
|
||||
|
||||
if storage_type == "minio":
|
||||
return MinioStorage(storage_config)
|
||||
elif storage_type == "cos":
|
||||
return CosStorage(storage_config)
|
||||
elif storage_type == "local":
|
||||
return LocalStorage(storage_config or {})
|
||||
return LocalStorage(storage_config)
|
||||
elif storage_type == "base64":
|
||||
return Base64Storage()
|
||||
return DummyStorage()
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
import logging
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextParser(BaseParser):
|
||||
"""
|
||||
Text document parser for processing plain text files.
|
||||
This parser handles text extraction and chunking from plain text documents.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse text document content by decoding bytes to string.
|
||||
|
||||
This is a straightforward parser that simply converts the binary content
|
||||
to text using appropriate character encoding.
|
||||
|
||||
Args:
|
||||
content: Raw document content as bytes
|
||||
|
||||
Returns:
|
||||
Parsed text content as string
|
||||
"""
|
||||
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
||||
text = endecode.decode_bytes(content)
|
||||
logger.info(
|
||||
f"Successfully parsed text document, extracted {len(text)} characters"
|
||||
)
|
||||
return Document(content=text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sample text for testing
|
||||
text = """## 标题1
|
||||

|
||||
## 标题2
|
||||

|
||||
## 标题3
|
||||
"""
|
||||
logger.info(f"Test text content: {text}")
|
||||
|
||||
# Define separators for text splitting
|
||||
seperators = ["\n\n", "\n", "。"]
|
||||
parser = TextParser(separators=seperators)
|
||||
logger.info("Splitting text into units")
|
||||
units = parser._split_into_units(text)
|
||||
logger.info(f"Split text into {len(units)} units")
|
||||
logger.info(f"Units: {units}")
|
||||
|
|
@ -4,86 +4,56 @@ package docreader;
|
|||
|
||||
option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
|
||||
|
||||
// 文档读取服务
|
||||
service DocReader {
|
||||
// 从文件读取文档
|
||||
rpc ReadFromFile(ReadFromFileRequest) returns (ReadResponse) {}
|
||||
// 从URL读取文档
|
||||
rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {}
|
||||
}
|
||||
|
||||
// 对象存储提供方
|
||||
enum StorageProvider {
|
||||
STORAGE_PROVIDER_UNSPECIFIED = 0;
|
||||
COS = 1; // 腾讯云 COS
|
||||
MINIO = 2; // MinIO/S3 兼容
|
||||
}
|
||||
|
||||
// 通用对象存储配置,兼容 COS 与 MinIO
|
||||
message StorageConfig {
|
||||
StorageProvider provider = 1; // 存储提供方
|
||||
string region = 2; // 区域(COS 使用)
|
||||
string bucket_name = 3; // 桶名
|
||||
string access_key_id = 4; // 访问密钥 ID(MinIO/S3 使用)
|
||||
string secret_access_key = 5; // 访问密钥 Secret(MinIO/S3 使用)
|
||||
string app_id = 6; // 应用 ID(COS 使用)
|
||||
string path_prefix = 7; // 路径前缀
|
||||
}
|
||||
|
||||
// VLM 配置
|
||||
message VLMConfig {
|
||||
string model_name = 1; // VLM Model Name
|
||||
string base_url = 2; // VLM Base URL
|
||||
string api_key = 3; // VLM API Key
|
||||
string interface_type = 4; // VLM Interface Type: "ollama" or "openai"
|
||||
rpc Read(ReadRequest) returns (ReadResponse) {}
|
||||
rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
|
||||
}
|
||||
|
||||
message ReadConfig {
|
||||
int32 chunk_size = 1; // 分块大小
|
||||
int32 chunk_overlap = 2; // 分块重叠
|
||||
repeated string separators = 3; // 分隔符
|
||||
bool enable_multimodal = 4; // 多模态处理
|
||||
StorageConfig storage_config = 5; // 对象存储配置(通用)
|
||||
VLMConfig vlm_config = 6; // VLM 配置
|
||||
string parser_engine = 1;
|
||||
map<string, string> parser_engine_overrides = 2;
|
||||
map<string, string> image_storage = 3; // storage provider config (provider, endpoint, bucket, ...)
|
||||
}
|
||||
|
||||
// 从文件读取文档请求
|
||||
message ReadFromFileRequest {
|
||||
bytes file_content = 1; // 文件内容
|
||||
string file_name = 2; // 文件名
|
||||
string file_type = 3; // 文件类型
|
||||
ReadConfig read_config = 4;
|
||||
string request_id = 5;
|
||||
// Unified read request: set file_content for file mode, url for URL mode.
|
||||
message ReadRequest {
|
||||
bytes file_content = 1;
|
||||
string file_name = 2;
|
||||
string file_type = 3;
|
||||
string url = 4;
|
||||
string title = 5;
|
||||
ReadConfig config = 6;
|
||||
string request_id = 7;
|
||||
}
|
||||
|
||||
// 从URL读取文档请求
|
||||
message ReadFromURLRequest {
|
||||
string url = 1; // 文档URL
|
||||
string title = 2; // 标题
|
||||
ReadConfig read_config = 3;
|
||||
string request_id = 4;
|
||||
message ImageRef {
|
||||
string filename = 1;
|
||||
string original_ref = 2;
|
||||
string mime_type = 3;
|
||||
string storage_key = 4; // download URL from shared storage
|
||||
bytes image_data = 5; // inline bytes fallback
|
||||
}
|
||||
|
||||
// 图片信息
|
||||
message Image {
|
||||
string url = 1; // 图片URL
|
||||
string caption = 2; // 图片描述
|
||||
string ocr_text = 3; // OCR提取的文本内容
|
||||
string original_url = 4; // 原始图片URL
|
||||
int32 start = 5; // 图片在文本中的开始位置
|
||||
int32 end = 6; // 图片在文本中的结束位置
|
||||
}
|
||||
|
||||
message Chunk {
|
||||
string content = 1; // 块内容
|
||||
int32 seq = 2; // 块在文档中的次序
|
||||
int32 start = 3; // 块在文档中的起始位置
|
||||
int32 end = 4; // 块在文档中的结束位置
|
||||
repeated Image images = 5; // 块中包含的图片信息
|
||||
}
|
||||
|
||||
// 从URL读取文档响应
|
||||
message ReadResponse {
|
||||
repeated Chunk chunks = 1; // 文档分块
|
||||
string error = 2; // 错误信息
|
||||
}
|
||||
string markdown_content = 1;
|
||||
repeated ImageRef image_refs = 2;
|
||||
string image_dir_path = 3;
|
||||
map<string, string> metadata = 4;
|
||||
string error = 5;
|
||||
}
|
||||
|
||||
message ListEnginesRequest {
|
||||
map<string, string> config_overrides = 1;
|
||||
}
|
||||
|
||||
message ParserEngineInfo {
|
||||
string name = 1;
|
||||
string description = 2;
|
||||
repeated string file_types = 3;
|
||||
bool available = 4;
|
||||
string unavailable_reason = 5;
|
||||
}
|
||||
|
||||
message ListEnginesResponse {
|
||||
repeated ParserEngineInfo engines = 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.5.1
|
||||
// - protoc v5.29.3
|
||||
// - protoc v6.33.4
|
||||
// source: docreader.proto
|
||||
|
||||
package proto
|
||||
|
|
@ -19,20 +19,18 @@ import (
|
|||
const _ = grpc.SupportPackageIsVersion9
|
||||
|
||||
const (
|
||||
DocReader_ReadFromFile_FullMethodName = "/docreader.DocReader/ReadFromFile"
|
||||
DocReader_ReadFromURL_FullMethodName = "/docreader.DocReader/ReadFromURL"
|
||||
DocReader_Read_FullMethodName = "/docreader.DocReader/Read"
|
||||
DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
|
||||
DocReader_ConvertToPDF_FullMethodName = "/docreader.DocReader/ConvertToPDF"
|
||||
)
|
||||
|
||||
// DocReaderClient is the client API for DocReader service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
//
|
||||
// 文档读取服务
|
||||
type DocReaderClient interface {
|
||||
// 从文件读取文档
|
||||
ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
// 从URL读取文档
|
||||
ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error)
|
||||
ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error)
|
||||
}
|
||||
|
||||
type docReaderClient struct {
|
||||
|
|
@ -43,20 +41,30 @@ func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient {
|
|||
return &docReaderClient{cc}
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
func (c *docReaderClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ReadFromFile_FullMethodName, in, out, cOpts...)
|
||||
err := c.cc.Invoke(ctx, DocReader_Read_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ReadFromURL_FullMethodName, in, out, cOpts...)
|
||||
out := new(ListEnginesResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ListEngines_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ConvertToPDFResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ConvertToPDF_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -66,13 +74,10 @@ func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLReques
|
|||
// DocReaderServer is the server API for DocReader service.
|
||||
// All implementations must embed UnimplementedDocReaderServer
|
||||
// for forward compatibility.
|
||||
//
|
||||
// 文档读取服务
|
||||
type DocReaderServer interface {
|
||||
// 从文件读取文档
|
||||
ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error)
|
||||
// 从URL读取文档
|
||||
ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error)
|
||||
Read(context.Context, *ReadRequest) (*ReadResponse, error)
|
||||
ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error)
|
||||
ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error)
|
||||
mustEmbedUnimplementedDocReaderServer()
|
||||
}
|
||||
|
||||
|
|
@ -83,11 +88,14 @@ type DocReaderServer interface {
|
|||
// pointer dereference when methods are called.
|
||||
type UnimplementedDocReaderServer struct{}
|
||||
|
||||
func (UnimplementedDocReaderServer) ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadFromFile not implemented")
|
||||
func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadFromURL not implemented")
|
||||
func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ConvertToPDF not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
|
||||
func (UnimplementedDocReaderServer) testEmbeddedByValue() {}
|
||||
|
|
@ -110,38 +118,56 @@ func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) {
|
|||
s.RegisterService(&DocReader_ServiceDesc, srv)
|
||||
}
|
||||
|
||||
func _DocReader_ReadFromFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadFromFileRequest)
|
||||
func _DocReader_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ReadFromFile(ctx, in)
|
||||
return srv.(DocReaderServer).Read(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ReadFromFile_FullMethodName,
|
||||
FullMethod: DocReader_Read_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ReadFromFile(ctx, req.(*ReadFromFileRequest))
|
||||
return srv.(DocReaderServer).Read(ctx, req.(*ReadRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DocReader_ReadFromURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadFromURLRequest)
|
||||
func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ListEnginesRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ReadFromURL(ctx, in)
|
||||
return srv.(DocReaderServer).ListEngines(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ReadFromURL_FullMethodName,
|
||||
FullMethod: DocReader_ListEngines_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ReadFromURL(ctx, req.(*ReadFromURLRequest))
|
||||
return srv.(DocReaderServer).ListEngines(ctx, req.(*ListEnginesRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DocReader_ConvertToPDF_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ConvertToPDFRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ConvertToPDF(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ConvertToPDF_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ConvertToPDF(ctx, req.(*ConvertToPDFRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
|
@ -154,12 +180,16 @@ var DocReader_ServiceDesc = grpc.ServiceDesc{
|
|||
HandlerType: (*DocReaderServer)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{
|
||||
MethodName: "ReadFromFile",
|
||||
Handler: _DocReader_ReadFromFile_Handler,
|
||||
MethodName: "Read",
|
||||
Handler: _DocReader_Read_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ReadFromURL",
|
||||
Handler: _DocReader_ReadFromURL_Handler,
|
||||
MethodName: "ListEngines",
|
||||
Handler: _DocReader_ListEngines_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ConvertToPDF",
|
||||
Handler: _DocReader_ConvertToPDF_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
|
|||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xb9\x01\n\rStorageConfig\x12,\n\x08provider\x18\x01 \x01(\x0e\x32\x1a.docreader.StorageProvider\x12\x0e\n\x06region\x18\x02 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x03 \x01(\t\x12\x15\n\raccess_key_id\x18\x04 \x01(\t\x12\x19\n\x11secret_access_key\x18\x05 \x01(\t\x12\x0e\n\x06\x61pp_id\x18\x06 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x07 \x01(\t\"Z\n\tVLMConfig\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x10\n\x08\x62\x61se_url\x18\x02 \x01(\t\x12\x0f\n\x07\x61pi_key\x18\x03 \x01(\t\x12\x16\n\x0einterface_type\x18\x04 \x01(\t\"\xc2\x01\n\nReadConfig\x12\x12\n\nchunk_size\x18\x01 \x01(\x05\x12\x15\n\rchunk_overlap\x18\x02 \x01(\x05\x12\x12\n\nseparators\x18\x03 \x03(\t\x12\x19\n\x11\x65nable_multimodal\x18\x04 \x01(\x08\x12\x30\n\x0estorage_config\x18\x05 \x01(\x0b\x32\x18.docreader.StorageConfig\x12(\n\nvlm_config\x18\x06 \x01(\x0b\x32\x14.docreader.VLMConfig\"\x91\x01\n\x13ReadFromFileRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12*\n\x0bread_config\x18\x04 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"p\n\x12ReadFromURLRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12*\n\x0bread_config\x18\x03 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"i\n\x05Image\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x61ption\x18\x02 \x01(\t\x12\x10\n\x08ocr_text\x18\x03 \x01(\t\x12\x14\n\x0coriginal_url\x18\x04 \x01(\t\x12\r\n\x05start\x18\x05 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x06 \x01(\x05\"c\n\x05\x43hunk\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x05\x12\r\n\x05start\x18\x03 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x05\x12 \n\x06images\x18\x05 \x03(\x0b\x32\x10.docreader.Image\"?\n\x0cReadResponse\x12 \n\x06\x63hunks\x18\x01 \x03(\x0b\x32\x10.docreader.Chunk\x12\r\n\x05\x65rror\x18\x02 \x01(\t*G\n\x0fStorageProvider\x12 \n\x1cSTORAGE_PROVIDER_UNSPECIFIED\x10\x00\x12\x07\n\x03\x43OS\x10\x01\x12\t\n\x05MINIO\x10\x02\x32\x9f\x01\n\tDocReader\x12I\n\x0cReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\x0bReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xa9\x02\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x12>\n\rimage_storage\x18\x03 \x03(\x0b\x32\'.docreader.ReadConfig.ImageStorageEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11ImageStorageEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo\"Q\n\x13\x43onvertToPDFRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\":\n\x14\x43onvertToPDFResponse\x12\x13\n\x0bpdf_content\x18\x01 \x01(\x0c\x12\r\n\x05\x65rror\x18\x02 \x01(\t2\xe9\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x12Q\n\x0c\x43onvertToPDF\x12\x1e.docreader.ConvertToPDFRequest\x1a\x1f.docreader.ConvertToPDFResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
|
|
@ -32,24 +32,40 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docreader_pb2', _globals)
|
|||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
_globals['DESCRIPTOR']._loaded_options = None
|
||||
_globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
|
||||
_globals['_STORAGEPROVIDER']._serialized_start=1042
|
||||
_globals['_STORAGEPROVIDER']._serialized_end=1113
|
||||
_globals['_STORAGECONFIG']._serialized_start=31
|
||||
_globals['_STORAGECONFIG']._serialized_end=216
|
||||
_globals['_VLMCONFIG']._serialized_start=218
|
||||
_globals['_VLMCONFIG']._serialized_end=308
|
||||
_globals['_READCONFIG']._serialized_start=311
|
||||
_globals['_READCONFIG']._serialized_end=505
|
||||
_globals['_READFROMFILEREQUEST']._serialized_start=508
|
||||
_globals['_READFROMFILEREQUEST']._serialized_end=653
|
||||
_globals['_READFROMURLREQUEST']._serialized_start=655
|
||||
_globals['_READFROMURLREQUEST']._serialized_end=767
|
||||
_globals['_IMAGE']._serialized_start=769
|
||||
_globals['_IMAGE']._serialized_end=874
|
||||
_globals['_CHUNK']._serialized_start=876
|
||||
_globals['_CHUNK']._serialized_end=975
|
||||
_globals['_READRESPONSE']._serialized_start=977
|
||||
_globals['_READRESPONSE']._serialized_end=1040
|
||||
_globals['_DOCREADER']._serialized_start=1116
|
||||
_globals['_DOCREADER']._serialized_end=1275
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._loaded_options = None
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_options = b'8\001'
|
||||
_globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
|
||||
_globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
|
||||
_globals['_READCONFIG']._serialized_start=31
|
||||
_globals['_READCONFIG']._serialized_end=328
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=215
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=275
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_start=277
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_end=328
|
||||
_globals['_READREQUEST']._serialized_start=331
|
||||
_globals['_READREQUEST']._serialized_end=491
|
||||
_globals['_IMAGEREF']._serialized_start=493
|
||||
_globals['_IMAGEREF']._serialized_end=603
|
||||
_globals['_READRESPONSE']._serialized_start=606
|
||||
_globals['_READRESPONSE']._serialized_end=832
|
||||
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=785
|
||||
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=832
|
||||
_globals['_LISTENGINESREQUEST']._serialized_start=835
|
||||
_globals['_LISTENGINESREQUEST']._serialized_end=989
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=935
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=989
|
||||
_globals['_PARSERENGINEINFO']._serialized_start=991
|
||||
_globals['_PARSERENGINEINFO']._serialized_end=1111
|
||||
_globals['_LISTENGINESRESPONSE']._serialized_start=1113
|
||||
_globals['_LISTENGINESRESPONSE']._serialized_end=1180
|
||||
_globals['_CONVERTTOPDFREQUEST']._serialized_start=1182
|
||||
_globals['_CONVERTTOPDFREQUEST']._serialized_end=1263
|
||||
_globals['_CONVERTTOPDFRESPONSE']._serialized_start=1265
|
||||
_globals['_CONVERTTOPDFRESPONSE']._serialized_end=1323
|
||||
_globals['_DOCREADER']._serialized_start=1326
|
||||
_globals['_DOCREADER']._serialized_end=1559
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from google.protobuf.internal import containers as _containers
|
||||
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from collections.abc import Iterable as _Iterable, Mapping as _Mapping
|
||||
|
|
@ -7,121 +6,130 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
|
|||
|
||||
DESCRIPTOR: _descriptor.FileDescriptor
|
||||
|
||||
class StorageProvider(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
STORAGE_PROVIDER_UNSPECIFIED: _ClassVar[StorageProvider]
|
||||
COS: _ClassVar[StorageProvider]
|
||||
MINIO: _ClassVar[StorageProvider]
|
||||
STORAGE_PROVIDER_UNSPECIFIED: StorageProvider
|
||||
COS: StorageProvider
|
||||
MINIO: StorageProvider
|
||||
|
||||
class StorageConfig(_message.Message):
|
||||
__slots__ = ("provider", "region", "bucket_name", "access_key_id", "secret_access_key", "app_id", "path_prefix")
|
||||
PROVIDER_FIELD_NUMBER: _ClassVar[int]
|
||||
REGION_FIELD_NUMBER: _ClassVar[int]
|
||||
BUCKET_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
ACCESS_KEY_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
SECRET_ACCESS_KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
APP_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
PATH_PREFIX_FIELD_NUMBER: _ClassVar[int]
|
||||
provider: StorageProvider
|
||||
region: str
|
||||
bucket_name: str
|
||||
access_key_id: str
|
||||
secret_access_key: str
|
||||
app_id: str
|
||||
path_prefix: str
|
||||
def __init__(self, provider: _Optional[_Union[StorageProvider, str]] = ..., region: _Optional[str] = ..., bucket_name: _Optional[str] = ..., access_key_id: _Optional[str] = ..., secret_access_key: _Optional[str] = ..., app_id: _Optional[str] = ..., path_prefix: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class VLMConfig(_message.Message):
|
||||
__slots__ = ("model_name", "base_url", "api_key", "interface_type")
|
||||
MODEL_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
BASE_URL_FIELD_NUMBER: _ClassVar[int]
|
||||
API_KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
INTERFACE_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
model_name: str
|
||||
base_url: str
|
||||
api_key: str
|
||||
interface_type: str
|
||||
def __init__(self, model_name: _Optional[str] = ..., base_url: _Optional[str] = ..., api_key: _Optional[str] = ..., interface_type: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ReadConfig(_message.Message):
|
||||
__slots__ = ("chunk_size", "chunk_overlap", "separators", "enable_multimodal", "storage_config", "vlm_config")
|
||||
CHUNK_SIZE_FIELD_NUMBER: _ClassVar[int]
|
||||
CHUNK_OVERLAP_FIELD_NUMBER: _ClassVar[int]
|
||||
SEPARATORS_FIELD_NUMBER: _ClassVar[int]
|
||||
ENABLE_MULTIMODAL_FIELD_NUMBER: _ClassVar[int]
|
||||
STORAGE_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
VLM_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
chunk_size: int
|
||||
chunk_overlap: int
|
||||
separators: _containers.RepeatedScalarFieldContainer[str]
|
||||
enable_multimodal: bool
|
||||
storage_config: StorageConfig
|
||||
vlm_config: VLMConfig
|
||||
def __init__(self, chunk_size: _Optional[int] = ..., chunk_overlap: _Optional[int] = ..., separators: _Optional[_Iterable[str]] = ..., enable_multimodal: bool = ..., storage_config: _Optional[_Union[StorageConfig, _Mapping]] = ..., vlm_config: _Optional[_Union[VLMConfig, _Mapping]] = ...) -> None: ...
|
||||
__slots__ = ("parser_engine", "parser_engine_overrides", "image_storage")
|
||||
class ParserEngineOverridesEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
class ImageStorageEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
PARSER_ENGINE_FIELD_NUMBER: _ClassVar[int]
|
||||
PARSER_ENGINE_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_STORAGE_FIELD_NUMBER: _ClassVar[int]
|
||||
parser_engine: str
|
||||
parser_engine_overrides: _containers.ScalarMap[str, str]
|
||||
image_storage: _containers.ScalarMap[str, str]
|
||||
def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ..., image_storage: _Optional[_Mapping[str, str]] = ...) -> None: ...
|
||||
|
||||
class ReadFromFileRequest(_message.Message):
|
||||
__slots__ = ("file_content", "file_name", "file_type", "read_config", "request_id")
|
||||
class ReadRequest(_message.Message):
|
||||
__slots__ = ("file_content", "file_name", "file_type", "url", "title", "config", "request_id")
|
||||
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
URL_FIELD_NUMBER: _ClassVar[int]
|
||||
TITLE_FIELD_NUMBER: _ClassVar[int]
|
||||
CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
file_content: bytes
|
||||
file_name: str
|
||||
file_type: str
|
||||
read_config: ReadConfig
|
||||
request_id: str
|
||||
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ReadFromURLRequest(_message.Message):
|
||||
__slots__ = ("url", "title", "read_config", "request_id")
|
||||
URL_FIELD_NUMBER: _ClassVar[int]
|
||||
TITLE_FIELD_NUMBER: _ClassVar[int]
|
||||
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
url: str
|
||||
title: str
|
||||
read_config: ReadConfig
|
||||
config: ReadConfig
|
||||
request_id: str
|
||||
def __init__(self, url: _Optional[str] = ..., title: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
|
||||
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., url: _Optional[str] = ..., title: _Optional[str] = ..., config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class Image(_message.Message):
|
||||
__slots__ = ("url", "caption", "ocr_text", "original_url", "start", "end")
|
||||
URL_FIELD_NUMBER: _ClassVar[int]
|
||||
CAPTION_FIELD_NUMBER: _ClassVar[int]
|
||||
OCR_TEXT_FIELD_NUMBER: _ClassVar[int]
|
||||
ORIGINAL_URL_FIELD_NUMBER: _ClassVar[int]
|
||||
START_FIELD_NUMBER: _ClassVar[int]
|
||||
END_FIELD_NUMBER: _ClassVar[int]
|
||||
url: str
|
||||
caption: str
|
||||
ocr_text: str
|
||||
original_url: str
|
||||
start: int
|
||||
end: int
|
||||
def __init__(self, url: _Optional[str] = ..., caption: _Optional[str] = ..., ocr_text: _Optional[str] = ..., original_url: _Optional[str] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class Chunk(_message.Message):
|
||||
__slots__ = ("content", "seq", "start", "end", "images")
|
||||
CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
SEQ_FIELD_NUMBER: _ClassVar[int]
|
||||
START_FIELD_NUMBER: _ClassVar[int]
|
||||
END_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGES_FIELD_NUMBER: _ClassVar[int]
|
||||
content: str
|
||||
seq: int
|
||||
start: int
|
||||
end: int
|
||||
images: _containers.RepeatedCompositeFieldContainer[Image]
|
||||
def __init__(self, content: _Optional[str] = ..., seq: _Optional[int] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., images: _Optional[_Iterable[_Union[Image, _Mapping]]] = ...) -> None: ...
|
||||
class ImageRef(_message.Message):
|
||||
__slots__ = ("filename", "original_ref", "mime_type", "storage_key", "image_data")
|
||||
FILENAME_FIELD_NUMBER: _ClassVar[int]
|
||||
ORIGINAL_REF_FIELD_NUMBER: _ClassVar[int]
|
||||
MIME_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
STORAGE_KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_DATA_FIELD_NUMBER: _ClassVar[int]
|
||||
filename: str
|
||||
original_ref: str
|
||||
mime_type: str
|
||||
storage_key: str
|
||||
image_data: bytes
|
||||
def __init__(self, filename: _Optional[str] = ..., original_ref: _Optional[str] = ..., mime_type: _Optional[str] = ..., storage_key: _Optional[str] = ..., image_data: _Optional[bytes] = ...) -> None: ...
|
||||
|
||||
class ReadResponse(_message.Message):
|
||||
__slots__ = ("chunks", "error")
|
||||
CHUNKS_FIELD_NUMBER: _ClassVar[int]
|
||||
__slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error")
|
||||
class MetadataEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_REFS_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
|
||||
METADATA_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
chunks: _containers.RepeatedCompositeFieldContainer[Chunk]
|
||||
markdown_content: str
|
||||
image_refs: _containers.RepeatedCompositeFieldContainer[ImageRef]
|
||||
image_dir_path: str
|
||||
metadata: _containers.ScalarMap[str, str]
|
||||
error: str
|
||||
def __init__(self, chunks: _Optional[_Iterable[_Union[Chunk, _Mapping]]] = ..., error: _Optional[str] = ...) -> None: ...
|
||||
def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ListEnginesRequest(_message.Message):
|
||||
__slots__ = ("config_overrides",)
|
||||
class ConfigOverridesEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
CONFIG_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
|
||||
config_overrides: _containers.ScalarMap[str, str]
|
||||
def __init__(self, config_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
|
||||
|
||||
class ParserEngineInfo(_message.Message):
|
||||
__slots__ = ("name", "description", "file_types", "available", "unavailable_reason")
|
||||
NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_TYPES_FIELD_NUMBER: _ClassVar[int]
|
||||
AVAILABLE_FIELD_NUMBER: _ClassVar[int]
|
||||
UNAVAILABLE_REASON_FIELD_NUMBER: _ClassVar[int]
|
||||
name: str
|
||||
description: str
|
||||
file_types: _containers.RepeatedScalarFieldContainer[str]
|
||||
available: bool
|
||||
unavailable_reason: str
|
||||
def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., file_types: _Optional[_Iterable[str]] = ..., available: bool = ..., unavailable_reason: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ListEnginesResponse(_message.Message):
|
||||
__slots__ = ("engines",)
|
||||
ENGINES_FIELD_NUMBER: _ClassVar[int]
|
||||
engines: _containers.RepeatedCompositeFieldContainer[ParserEngineInfo]
|
||||
def __init__(self, engines: _Optional[_Iterable[_Union[ParserEngineInfo, _Mapping]]] = ...) -> None: ...
|
||||
|
||||
class ConvertToPDFRequest(_message.Message):
|
||||
__slots__ = ("file_content", "file_name", "file_type")
|
||||
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
file_content: bytes
|
||||
file_name: str
|
||||
file_type: str
|
||||
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ConvertToPDFResponse(_message.Message):
|
||||
__slots__ = ("pdf_content", "error")
|
||||
PDF_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
pdf_content: bytes
|
||||
error: str
|
||||
def __init__(self, pdf_content: _Optional[bytes] = ..., error: _Optional[str] = ...) -> None: ...
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
import grpc
|
||||
import warnings
|
||||
|
||||
import docreader_pb2 as docreader__pb2
|
||||
from docreader.proto import docreader_pb2 as docreader__pb2
|
||||
|
||||
GRPC_GENERATED_VERSION = '1.76.0'
|
||||
GRPC_GENERATED_VERSION = '1.78.0'
|
||||
GRPC_VERSION = grpc.__version__
|
||||
_version_not_supported = False
|
||||
|
||||
|
|
@ -26,8 +26,7 @@ if _version_not_supported:
|
|||
|
||||
|
||||
class DocReaderStub(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def __init__(self, channel):
|
||||
"""Constructor.
|
||||
|
|
@ -35,32 +34,40 @@ class DocReaderStub(object):
|
|||
Args:
|
||||
channel: A grpc.Channel.
|
||||
"""
|
||||
self.ReadFromFile = channel.unary_unary(
|
||||
'/docreader.DocReader/ReadFromFile',
|
||||
request_serializer=docreader__pb2.ReadFromFileRequest.SerializeToString,
|
||||
self.Read = channel.unary_unary(
|
||||
'/docreader.DocReader/Read',
|
||||
request_serializer=docreader__pb2.ReadRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ReadResponse.FromString,
|
||||
_registered_method=True)
|
||||
self.ReadFromURL = channel.unary_unary(
|
||||
'/docreader.DocReader/ReadFromURL',
|
||||
request_serializer=docreader__pb2.ReadFromURLRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ReadResponse.FromString,
|
||||
self.ListEngines = channel.unary_unary(
|
||||
'/docreader.DocReader/ListEngines',
|
||||
request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ListEnginesResponse.FromString,
|
||||
_registered_method=True)
|
||||
self.ConvertToPDF = channel.unary_unary(
|
||||
'/docreader.DocReader/ConvertToPDF',
|
||||
request_serializer=docreader__pb2.ConvertToPDFRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ConvertToPDFResponse.FromString,
|
||||
_registered_method=True)
|
||||
|
||||
|
||||
class DocReaderServicer(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def ReadFromFile(self, request, context):
|
||||
"""从文件读取文档
|
||||
"""
|
||||
def Read(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def ReadFromURL(self, request, context):
|
||||
"""从URL读取文档
|
||||
"""
|
||||
def ListEngines(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def ConvertToPDF(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
|
@ -68,15 +75,20 @@ class DocReaderServicer(object):
|
|||
|
||||
def add_DocReaderServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
'ReadFromFile': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ReadFromFile,
|
||||
request_deserializer=docreader__pb2.ReadFromFileRequest.FromString,
|
||||
'Read': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Read,
|
||||
request_deserializer=docreader__pb2.ReadRequest.FromString,
|
||||
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
|
||||
),
|
||||
'ReadFromURL': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ReadFromURL,
|
||||
request_deserializer=docreader__pb2.ReadFromURLRequest.FromString,
|
||||
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
|
||||
'ListEngines': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ListEngines,
|
||||
request_deserializer=docreader__pb2.ListEnginesRequest.FromString,
|
||||
response_serializer=docreader__pb2.ListEnginesResponse.SerializeToString,
|
||||
),
|
||||
'ConvertToPDF': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ConvertToPDF,
|
||||
request_deserializer=docreader__pb2.ConvertToPDFRequest.FromString,
|
||||
response_serializer=docreader__pb2.ConvertToPDFResponse.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
|
|
@ -87,11 +99,10 @@ def add_DocReaderServicer_to_server(servicer, server):
|
|||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
class DocReader(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
@staticmethod
|
||||
def ReadFromFile(request,
|
||||
def Read(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
|
|
@ -104,8 +115,8 @@ class DocReader(object):
|
|||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/docreader.DocReader/ReadFromFile',
|
||||
docreader__pb2.ReadFromFileRequest.SerializeToString,
|
||||
'/docreader.DocReader/Read',
|
||||
docreader__pb2.ReadRequest.SerializeToString,
|
||||
docreader__pb2.ReadResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
|
|
@ -118,7 +129,7 @@ class DocReader(object):
|
|||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def ReadFromURL(request,
|
||||
def ListEngines(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
|
|
@ -131,9 +142,36 @@ class DocReader(object):
|
|||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/docreader.DocReader/ReadFromURL',
|
||||
docreader__pb2.ReadFromURLRequest.SerializeToString,
|
||||
docreader__pb2.ReadResponse.FromString,
|
||||
'/docreader.DocReader/ListEngines',
|
||||
docreader__pb2.ListEnginesRequest.SerializeToString,
|
||||
docreader__pb2.ListEnginesResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def ConvertToPDF(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/docreader.DocReader/ConvertToPDF',
|
||||
docreader__pb2.ConvertToPDFRequest.SerializeToString,
|
||||
docreader__pb2.ConvertToPDFResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/bash
|
||||
set -x
|
||||
set -ex
|
||||
|
||||
# 设置目录
|
||||
PROTO_DIR="docreader/proto"
|
||||
|
|
@ -13,12 +13,16 @@ python3 -m grpc_tools.protoc -I${PROTO_DIR} \
|
|||
--grpc_python_out=${PYTHON_OUT} \
|
||||
${PROTO_DIR}/docreader.proto
|
||||
|
||||
# 生成Go代码
|
||||
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
|
||||
--go_opt=paths=source_relative \
|
||||
--go-grpc_out=${GO_OUT} \
|
||||
--go-grpc_opt=paths=source_relative \
|
||||
${PROTO_DIR}/docreader.proto
|
||||
# 生成Go代码(仅在 protoc-gen-go 可用时执行)
|
||||
if command -v protoc-gen-go &> /dev/null; then
|
||||
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
|
||||
--go_opt=paths=source_relative \
|
||||
--go-grpc_out=${GO_OUT} \
|
||||
--go-grpc_opt=paths=source_relative \
|
||||
${PROTO_DIR}/docreader.proto
|
||||
else
|
||||
echo "protoc-gen-go not found, skipping Go code generation"
|
||||
fi
|
||||
|
||||
# 修复Python导入问题(MacOS兼容版本)
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
# WeKnora Lite
|
||||
|
||||
零外部依赖的单二进制部署模式。无需 Docker、PostgreSQL、Redis,适合快速体验和小规模私有部署。
|
||||
|
||||
## 架构
|
||||
|
||||
| 组件 | 标准版 | Lite 版 |
|
||||
|------|--------|---------|
|
||||
| 数据库 | PostgreSQL | SQLite (WAL) |
|
||||
| 向量检索 | pgvector / Qdrant / ES | sqlite-vec (vec0) |
|
||||
| 关键词检索 | ParadeDB BM25 / ES | SQLite FTS5 |
|
||||
| 消息队列 | Redis + Asynq | 内存 SyncTaskExecutor |
|
||||
| 会话存储 | Redis | 内存 |
|
||||
| 流管理 | Redis / 内存 | 内存 |
|
||||
| 文件存储 | MinIO / COS / 本地 | 本地 |
|
||||
| 文档解析 | DocReader (gRPC) | 不可用(文本/段落导入可用)|
|
||||
| 前端 | Nginx 容器 | Go 内置静态文件服务 |
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 方式一:Homebrew 安装(macOS / Linux,推荐)
|
||||
|
||||
```bash
|
||||
brew tap Tencent/weknora https://github.com/Tencent/WeKnora
|
||||
brew install weknora-lite
|
||||
```
|
||||
|
||||
安装完成后,推荐使用 **brew services** 以后台服务方式运行:
|
||||
|
||||
```bash
|
||||
brew services start weknora-lite # 启动服务(开机自动启动)
|
||||
brew services info weknora-lite # 查看运行状态
|
||||
# 首次启动自动创建配置文件 ~/.config/weknora/.env.lite
|
||||
# 数据存储在 ~/.local/share/weknora/
|
||||
# 访问 http://localhost:8080
|
||||
```
|
||||
|
||||
常用服务管理命令:
|
||||
|
||||
```bash
|
||||
brew services stop weknora-lite # 停止服务
|
||||
brew services restart weknora-lite # 重启服务(修改配置后需重启)
|
||||
brew services info weknora-lite # 查看状态
|
||||
```
|
||||
|
||||
日志位于 `$(brew --prefix)/var/log/weknora-lite.log`。
|
||||
|
||||
也可以前台直接运行:
|
||||
|
||||
```bash
|
||||
weknora-lite
|
||||
```
|
||||
|
||||
如需修改配置(LLM 服务地址、安全密钥等):
|
||||
|
||||
```bash
|
||||
$EDITOR ~/.config/weknora/.env.lite
|
||||
brew services restart weknora-lite # 修改配置后重启生效
|
||||
```
|
||||
|
||||
> **LLM 服务**:WeKnora Lite 需要一个 OpenAI 兼容的 LLM 服务来提供对话和 Embedding 能力。
|
||||
> 可以使用 [Ollama](https://ollama.com/)(本地)、通义千问、OpenAI 等任何兼容服务,
|
||||
> 在配置文件中设置对应的地址和 API Key 即可。
|
||||
|
||||
### 方式二:下载预编译包
|
||||
|
||||
从 [GitHub Releases](https://github.com/Tencent/WeKnora/releases) 下载对应平台的 tarball:
|
||||
|
||||
| 文件 | 平台 |
|
||||
|------|------|
|
||||
| `WeKnora-lite_*_linux_amd64.tar.gz` | Linux x86_64 |
|
||||
| `WeKnora-lite_*_linux_arm64.tar.gz` | Linux ARM64 |
|
||||
| `WeKnora-lite_*_darwin_amd64.tar.gz` | macOS Intel |
|
||||
| `WeKnora-lite_*_darwin_arm64.tar.gz` | macOS Apple Silicon |
|
||||
|
||||
```bash
|
||||
# 1. 解压
|
||||
tar xzf WeKnora-lite_v0.2.0_darwin_arm64.tar.gz
|
||||
cd WeKnora-lite_v0.2.0_darwin_arm64
|
||||
|
||||
# 2. 配置
|
||||
cp .env.lite.example .env.lite
|
||||
# 编辑 .env.lite,配置 LLM 服务地址和安全密钥
|
||||
|
||||
# 3. 运行
|
||||
set -a && source .env.lite && set +a
|
||||
./WeKnora-lite
|
||||
# 访问 http://localhost:8080
|
||||
```
|
||||
|
||||
### 方式三:从源码构建
|
||||
|
||||
前置条件:Go 1.22+(需要 CGO)、C 编译器 (gcc/clang)、Node.js 22+(前端构建)。
|
||||
|
||||
```bash
|
||||
make run-lite
|
||||
```
|
||||
|
||||
## 配置
|
||||
|
||||
Lite 模式通过 `.env.lite` 文件配置(模板见 `.env.lite.example`)。关键环境变量:
|
||||
|
||||
```bash
|
||||
DB_DRIVER=sqlite # 使用 SQLite
|
||||
DB_PATH=./data/weknora.db # 数据库文件路径
|
||||
RETRIEVE_DRIVER=sqlite # SQLite 检索引擎 (FTS5 + sqlite-vec)
|
||||
STORAGE_TYPE=local # 本地文件存储
|
||||
LOCAL_STORAGE_BASE_DIR=./data/files
|
||||
STREAM_MANAGER_TYPE=memory # 内存流管理
|
||||
# REDIS_ADDR= # 留空 = 不使用 Redis
|
||||
OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
```
|
||||
|
||||
完整配置参见 [.env.lite.example](../.env.lite.example)。
|
||||
|
||||
## 后台运行
|
||||
|
||||
### Homebrew 用户(macOS / Linux)
|
||||
|
||||
Homebrew 安装后直接使用 `brew services` 管理,详见上方「快速开始 → 方式一」。
|
||||
|
||||
### Linux systemd(tarball 安装)
|
||||
|
||||
tarball 中附带 `weknora-lite.service` 模板,按需修改路径后安装:
|
||||
|
||||
```bash
|
||||
# 创建用户和目录
|
||||
sudo useradd -r -s /sbin/nologin weknora
|
||||
sudo mkdir -p /opt/weknora/data
|
||||
sudo cp WeKnora-lite web/ .env.lite /opt/weknora/
|
||||
sudo chown -R weknora:weknora /opt/weknora
|
||||
|
||||
# 安装并启动服务
|
||||
sudo cp weknora-lite.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now weknora-lite
|
||||
|
||||
# 管理
|
||||
sudo systemctl status weknora-lite # 查看状态
|
||||
sudo journalctl -u weknora-lite -f # 查看日志
|
||||
```
|
||||
|
||||
## 功能限制
|
||||
|
||||
与标准版相比,Lite 版有以下限制:
|
||||
|
||||
- **文档解析**:不支持文件上传和 URL 导入的自动解析(PDF/Word/Excel 等)。可使用文本和段落方式手动导入。
|
||||
- **向量检索**:sqlite-vec 使用精确 KNN(非近似),适合 10 万条以下的小规模数据集。
|
||||
- **并发**:SQLite 单写者模型,高并发写入场景下性能不如 PostgreSQL。
|
||||
- **任务队列**:无持久化队列,进程重启后未完成的异步任务会丢失。
|
||||
- **知识图谱**:默认禁用 (`NEO4J_ENABLE=false`)。
|
||||
- **Agent Skills 沙箱**:默认禁用 (`WEKNORA_SANDBOX_MODE=disabled`)。
|
||||
|
||||
## 数据目录
|
||||
|
||||
默认所有数据存储在 `./data/` 目录下:
|
||||
|
||||
```
|
||||
data/
|
||||
├── weknora.db # SQLite 数据库
|
||||
├── weknora.db-wal # WAL 日志
|
||||
└── files/ # 上传文件
|
||||
```
|
||||
|
||||
备份只需复制整个 `data/` 目录。
|
||||
|
|
@ -21,6 +21,15 @@ server {
|
|||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# 本地存储文件代理到后端服务(用于渲染 markdown 中的图片)
|
||||
location /files/ {
|
||||
proxy_pass ${APP_SCHEME}://${APP_HOST}:${APP_PORT}/files/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# API请求代理到后端服务
|
||||
# APP_SCHEME 默认 http,远程 HTTPS 后端可设为 https
|
||||
location /api/ {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@
|
|||
"@microsoft/fetch-event-source": "^2.0.1",
|
||||
"@types/dompurify": "^3.0.5",
|
||||
"@types/papaparse": "^5.5.0",
|
||||
"@vue-office/pptx": "^1.0.1",
|
||||
"axios": "^1.8.4",
|
||||
"docx-preview": "^0.3.7",
|
||||
"dompurify": "^3.2.6",
|
||||
"highlight.js": "^11.11.1",
|
||||
"marked": "^5.1.2",
|
||||
|
|
@ -27,6 +29,7 @@
|
|||
"tdesign-icons-vue-next": "^0.4.1",
|
||||
"tdesign-vue-next": "^1.17.2",
|
||||
"vue": "^3.5.13",
|
||||
"vue-demi": "^0.14.6",
|
||||
"vue-i18n": "^11.1.12",
|
||||
"vue-router": "^4.5.0",
|
||||
"webpack": "^5.94.0",
|
||||
|
|
@ -41,10 +44,10 @@
|
|||
"@vue/tsconfig": "^0.7.0",
|
||||
"less": "^4.3.0",
|
||||
"less-loader": "^12.2.0",
|
||||
"npm-run-all2": "^7.0.2",
|
||||
"npm-run-all2": "^8.0.4",
|
||||
"typescript": "~5.8.0",
|
||||
"vite": "^7.2.2",
|
||||
"vue-tsc": "^2.2.8"
|
||||
"vue-tsc": "^3.2.5"
|
||||
},
|
||||
"overrides": {
|
||||
"lightningcss": "none",
|
||||
|
|
|
|||
|
|
@ -82,24 +82,13 @@ export interface KBModelConfigRequest {
|
|||
chunkSize: number
|
||||
chunkOverlap: number
|
||||
separators: string[]
|
||||
parserEngineRules?: { file_types: string[]; engine: string }[]
|
||||
}
|
||||
multimodal: {
|
||||
enabled: boolean
|
||||
storageType?: 'cos' | 'minio'
|
||||
cos?: {
|
||||
secretId: string
|
||||
secretKey: string
|
||||
region: string
|
||||
bucketName: string
|
||||
appId: string
|
||||
pathPrefix: string
|
||||
}
|
||||
minio?: {
|
||||
bucketName: string
|
||||
useSSL: boolean
|
||||
pathPrefix: string
|
||||
}
|
||||
}
|
||||
/** 存储引擎选择:"local" | "minio" | "cos",影响文档上传与文档内图片存储 */
|
||||
storageProvider?: string
|
||||
nodeExtract: {
|
||||
enabled: boolean
|
||||
text: string
|
||||
|
|
|
|||
|
|
@ -106,6 +106,10 @@ export function downKnowledgeDetails(id: string) {
|
|||
return getDown(`/api/v1/knowledge/${id}/download`);
|
||||
}
|
||||
|
||||
export function previewKnowledgeFile(id: string) {
|
||||
return getDown(`/api/v1/knowledge/${id}/preview`);
|
||||
}
|
||||
|
||||
/** @param idsQueryString - query string with ids (e.g. ids=xxx&ids=yyy) */
|
||||
export function batchQueryKnowledge(idsQueryString: string, kbId?: string, agentId?: string) {
|
||||
let qs = idsQueryString;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { get, put } from '@/utils/request'
|
||||
import { get, post, put } from '@/utils/request'
|
||||
|
||||
export interface SystemInfo {
|
||||
version: string
|
||||
edition?: string
|
||||
commit_id?: string
|
||||
build_time?: string
|
||||
go_version?: string
|
||||
|
|
@ -110,3 +111,115 @@ export interface ListMinioBucketsResponse {
|
|||
export function listMinioBuckets(): Promise<{ data: ListMinioBucketsResponse }> {
|
||||
return get('/api/v1/system/minio/buckets')
|
||||
}
|
||||
|
||||
export interface ParserEngineInfo {
|
||||
Name: string
|
||||
Description: string
|
||||
FileTypes: string[]
|
||||
Available?: boolean
|
||||
UnavailableReason?: string
|
||||
}
|
||||
|
||||
/** 解析引擎配置(引擎相关存租户;docreader 地址由环境变量配置) */
|
||||
export interface ParserEngineConfig {
|
||||
docreader_addr?: string
|
||||
docreader_transport?: string
|
||||
mineru_endpoint?: string
|
||||
mineru_api_key?: string
|
||||
mineru_api_base_url?: string
|
||||
// MinerU 自建参数
|
||||
mineru_model?: string
|
||||
mineru_enable_formula?: boolean | null
|
||||
mineru_enable_table?: boolean | null
|
||||
mineru_enable_ocr?: boolean | null
|
||||
mineru_language?: string
|
||||
// MinerU 云 API 参数
|
||||
mineru_cloud_model?: string
|
||||
mineru_cloud_enable_formula?: boolean | null
|
||||
mineru_cloud_enable_table?: boolean | null
|
||||
mineru_cloud_enable_ocr?: boolean | null
|
||||
mineru_cloud_language?: string
|
||||
}
|
||||
|
||||
export interface ParserEnginesResponse {
|
||||
data: ParserEngineInfo[]
|
||||
docreader_addr?: string
|
||||
/** 连接方式:grpc | http,由服务端环境/配置决定 */
|
||||
docreader_transport?: string
|
||||
connected?: boolean
|
||||
}
|
||||
|
||||
export function getParserEngines(): Promise<ParserEnginesResponse> {
|
||||
return get('/api/v1/system/parser-engines')
|
||||
}
|
||||
|
||||
/** 使用当前填写的参数检测引擎可用性(不保存),用于填写新参数后即时测试 */
|
||||
export function checkParserEngines(config: ParserEngineConfig): Promise<ParserEnginesResponse> {
|
||||
return post('/api/v1/system/parser-engines/check', config)
|
||||
}
|
||||
|
||||
export function getParserEngineConfig(): Promise<{ data: ParserEngineConfig }> {
|
||||
return get('/api/v1/tenants/kv/parser-engine-config')
|
||||
}
|
||||
|
||||
export function updateParserEngineConfig(config: ParserEngineConfig): Promise<{ data: ParserEngineConfig }> {
|
||||
return put('/api/v1/tenants/kv/parser-engine-config', config)
|
||||
}
|
||||
|
||||
export function reconnectDocReader(addr: string): Promise<ParserEnginesResponse & { msg?: string }> {
|
||||
return post('/api/v1/system/docreader/reconnect', { addr })
|
||||
}
|
||||
|
||||
// ---- 存储引擎配置(租户级,供文档/图片存储与 docreader 使用) ----
|
||||
|
||||
export interface StorageEngineConfig {
|
||||
default_provider: string // "local" | "minio" | "cos"
|
||||
local?: { path_prefix: string }
|
||||
minio?: { mode: string; endpoint: string; access_key_id: string; secret_access_key: string; bucket_name: string; use_ssl: boolean; path_prefix: string }
|
||||
cos?: {
|
||||
secret_id: string
|
||||
secret_key: string
|
||||
region: string
|
||||
bucket_name: string
|
||||
app_id: string
|
||||
path_prefix: string
|
||||
}
|
||||
}
|
||||
|
||||
export interface StorageEngineStatusItem {
|
||||
name: string
|
||||
available: boolean
|
||||
description: string
|
||||
}
|
||||
|
||||
export interface GetStorageEngineStatusResponse {
|
||||
engines: StorageEngineStatusItem[]
|
||||
minio_env_available: boolean
|
||||
}
|
||||
|
||||
export function getStorageEngineConfig(): Promise<{ data: StorageEngineConfig }> {
|
||||
return get('/api/v1/tenants/kv/storage-engine-config')
|
||||
}
|
||||
|
||||
export function updateStorageEngineConfig(config: StorageEngineConfig): Promise<{ data: StorageEngineConfig }> {
|
||||
return put('/api/v1/tenants/kv/storage-engine-config', config)
|
||||
}
|
||||
|
||||
export function getStorageEngineStatus(): Promise<{ data: GetStorageEngineStatusResponse }> {
|
||||
return get('/api/v1/system/storage-engine-status')
|
||||
}
|
||||
|
||||
export interface StorageCheckRequest {
|
||||
provider: string // "minio" | "cos"
|
||||
minio?: StorageEngineConfig['minio']
|
||||
cos?: StorageEngineConfig['cos']
|
||||
}
|
||||
|
||||
export interface StorageCheckResponse {
|
||||
ok: boolean
|
||||
message: string
|
||||
}
|
||||
|
||||
export function checkStorageEngine(req: StorageCheckRequest): Promise<{ data: StorageCheckResponse }> {
|
||||
return post('/api/v1/system/storage-engine-check', req)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M10 3L10.8 6.2C10.9 6.7 11.3 7.1 11.8 7.2L15 8L11.8 8.8C11.3 8.9 10.9 9.3 10.8 9.8L10 13L9.2 9.8C9.1 9.3 8.7 8.9 8.2 8.8L5 8L8.2 7.2C8.7 7.1 9.1 6.7 9.2 6.2L10 3Z" fill="#000000" fill-opacity=".9" stroke="#000000" stroke-opacity=".9" stroke-width="0.8" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M15.5 4L15.8 5.2C15.85 5.45 16.05 5.65 16.3 5.7L17.5 6L16.3 6.3C16.05 6.35 15.85 6.55 15.8 6.8L15.5 8L15.2 6.8C15.15 6.55 14.95 6.35 14.7 6.3L13.5 6L14.7 5.7C14.95 5.65 15.15 5.45 15.2 5.2L15.5 4Z" fill="#000000" fill-opacity=".9" stroke="#000000" stroke-opacity=".9" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M4.5 13L4.8 14.2C4.85 14.45 5.05 14.65 5.3 14.7L6.5 15L5.3 15.3C5.05 15.35 4.85 15.55 4.8 15.8L4.5 17L4.2 15.8C4.15 15.55 3.95 15.35 3.7 15.3L2.5 15L3.7 14.7C3.95 14.65 4.15 14.45 4.2 14.2L4.5 13Z" fill="#000000" fill-opacity=".9" stroke="#000000" stroke-opacity=".9" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M10 3L10.8 6.2C10.9 6.7 11.3 7.1 11.8 7.2L15 8L11.8 8.8C11.3 8.9 10.9 9.3 10.8 9.8L10 13L9.2 9.8C9.1 9.3 8.7 8.9 8.2 8.8L5 8L8.2 7.2C8.7 7.1 9.1 6.7 9.2 6.2L10 3Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.8" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M15.5 4L15.8 5.2C15.85 5.45 16.05 5.65 16.3 5.7L17.5 6L16.3 6.3C16.05 6.35 15.85 6.55 15.8 6.8L15.5 8L15.2 6.8C15.15 6.55 14.95 6.35 14.7 6.3L13.5 6L14.7 5.7C14.95 5.65 15.15 5.45 15.2 5.2L15.5 4Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M4.5 13L4.8 14.2C4.85 14.45 5.05 14.65 5.3 14.7L6.5 15L5.3 15.3C5.05 15.35 4.85 15.55 4.8 15.8L4.5 17L4.2 15.8C4.15 15.55 3.95 15.35 3.7 15.3L2.5 15L3.7 14.7C3.95 14.65 4.15 14.45 4.2 14.2L4.5 13Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
|
@ -1,3 +1,3 @@
|
|||
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="#8A8A8A" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
|
||||
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="#000000" stroke-opacity=".4" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 421 B After Width: | Height: | Size: 441 B |
|
|
@ -1,3 +1,3 @@
|
|||
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
|
||||
<path d="M10 10C8.8 7.5 7.8 3.8 4.8 3.8C2.2 3.8 0.8 6.8 0.8 10C0.8 13.2 2.2 16.2 4.8 16.2C7.8 16.2 8.8 12.5 10 10C11.2 7.5 12.5 5.5 14.5 5.5C16.5 5.5 18 7.5 18 10C18 12.5 16.5 14.5 14.5 14.5C12.5 14.5 11.2 12.5 10 10Z" stroke="#000000" stroke-opacity=".6" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" fill="none"/>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 426 B After Width: | Height: | Size: 441 B |
|
|
@ -1,4 +1,4 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="45o4l8wsy__design-iconfont" width="128" height="128">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#07C05F" fill-opacity=".9"></path>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#07C05F" fill-opacity=".9"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 851 B After Width: | Height: | Size: 814 B |
|
|
@ -1,4 +1,4 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="45o4l8wsy__design-iconfont" width="128" height="128">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#737373" fill-opacity=".9"></path>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#737373" fill-opacity=".9"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 851 B After Width: | Height: | Size: 814 B |
|
|
@ -1,4 +1,4 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="45o4l8wsy__design-iconfont" width="128" height="128">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#000000" fill-opacity=".9"></path>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#000000" fill-opacity=".9"></path>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M1.875 2.5C1.875 2.15483 2.15482 1.875 2.5 1.875H17.5C17.8451 1.875 18.125 2.15483 18.125 2.5V13.75C18.125 14.0951 17.8451 14.375 17.5 14.375H9.04274L6.10514 17.9001C5.93669 18.1023 5.65965 18.1773 5.41224 18.0876C5.16481 17.9981 5 17.7631 5 17.5V14.375H2.5C2.15482 14.375 1.875 14.0951 1.875 13.75V2.5ZM3.125 3.125V13.125H5.625C5.97017 13.125 6.25 13.4049 6.25 13.75V15.7738L8.26986 13.3499C8.38861 13.2074 8.56451 13.125 8.75 13.125H16.875V3.125H3.125Z" fill="#000000" fill-opacity=".6"></path>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.375 5H10.625V8.44961L13.5154 10.762L12.7346 11.738L9.375 9.05039V5Z" fill="#000000" fill-opacity=".6"></path>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 851 B After Width: | Height: | Size: 814 B |
|
|
@ -1,3 +1,3 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="cvw0hqhj9__design-iconfont" width="128" height="128">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#07C05F"></path>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#07C05F" stroke="#07C05F" stroke-width="0.3"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.0 KiB After Width: | Height: | Size: 3.0 KiB |
|
|
@ -1,3 +1,3 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="cvw0hqhj9__design-iconfont" width="128" height="128">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#717171"></path>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" width="20" height="20">
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M9.17736 1.28207C9.32257 1.23367 9.4805 1.24024 9.6212 1.30054L18.3713 5.05054C18.5619 5.13222 18.6995 5.30316 18.7388 5.50681C18.778 5.71045 18.7136 5.9203 18.567 6.06694C18.5138 6.12012 18.4366 6.24314 18.3744 6.46076C18.3146 6.66994 18.2812 6.92361 18.2812 7.1875C18.2812 7.45139 18.3146 7.70506 18.3744 7.91424C18.4366 8.13186 18.5138 8.25487 18.567 8.30806C18.811 8.55214 18.811 8.94786 18.567 9.19194C18.5138 9.24512 18.4366 9.36814 18.3744 9.58576C18.3146 9.79494 18.2812 10.0486 18.2812 10.3125C18.2812 10.5764 18.3146 10.8301 18.3744 11.0392C18.4366 11.2569 18.5138 11.3799 18.567 11.4331C18.811 11.6771 18.811 12.0729 18.567 12.3169C18.5138 12.3701 18.4366 12.4931 18.3744 12.7108C18.3146 12.92 18.2812 13.1736 18.2812 13.4375C18.2812 13.7014 18.3146 13.955 18.3744 14.1642C18.4366 14.3819 18.5138 14.5049 18.567 14.558C18.709 14.7001 18.7741 14.9017 18.7419 15.1001C18.7097 15.2984 18.5843 15.4691 18.4045 15.559L12.1545 18.684C11.9886 18.767 11.7944 18.772 11.6245 18.6976L1.62449 14.3226C1.397 14.2231 1.25 13.9983 1.25 13.75V4.375C1.25 4.10599 1.42215 3.86715 1.67736 3.78207L9.17736 1.28207ZM2.5 5.33064L10.9868 9.04362C10.9627 9.10997 10.9413 9.1765 10.9225 9.24236C10.826 9.58006 10.7812 9.95139 10.7812 10.3125C10.7812 10.6736 10.826 11.0449 10.9225 11.3826C10.9694 11.5469 11.0321 11.7153 11.1153 11.875C11.0321 12.0347 10.9694 12.2031 10.9225 12.3674C10.826 12.705 10.7812 13.0764 10.7812 13.4375C10.7812 13.7986 10.826 14.17 10.9225 14.5076C10.9694 14.6719 11.0321 14.8403 11.1153 15C11.0321 15.1597 10.9694 15.3281 10.9225 15.4924C10.826 15.83 10.7812 16.2014 10.7812 16.5625C10.7812 16.6986 10.7876 16.8361 10.8007 16.9728L2.5 13.3413V5.33064ZM12.1275 17.3C12.1265 17.2964 12.1254 17.2929 12.1244 17.2892C12.0646 17.08 12.0312 16.8264 12.0312 16.5625C12.0312 16.2986 12.0646 16.045 12.1244 15.8358C12.1695 15.6779 12.2225 15.5697 12.2686 15.502L17.0432 13.1146C17.0351 13.2224 17.0312 13.3304 17.0312 13.4375C17.0312 13.7986 17.076 14.17 17.1725 14.5076C17.1944 14.5844 17.2198 14.662 17.249 14.7393L12.1275 17.3ZM17.249 11.6142L12.1275 14.175C12.1265 14.1714 12.1254 14.1679 12.1244 14.1642C12.0646 13.955 12.0312 13.7014 12.0312 13.4375C12.0312 13.1736 12.0646 12.92 12.1244 12.7108C12.1695 12.5529 12.2225 12.4447 12.2686 12.377L17.0432 9.98965C17.0351 10.0974 17.0312 10.2054 17.0312 10.3125C17.0312 10.6736 17.076 11.0449 17.1725 11.3826C17.1944 11.4594 17.2198 11.537 17.249 11.6142ZM17.249 8.48921L12.1275 11.05C12.1265 11.0464 12.1254 11.0428 12.1244 11.0392C12.0646 10.8301 12.0312 10.5764 12.0312 10.3125C12.0312 10.0486 12.0646 9.79494 12.1244 9.58576C12.1695 9.42785 12.2225 9.31975 12.2686 9.252L17.0432 6.86465C17.0351 6.9724 17.0312 7.0804 17.0312 7.1875C17.0312 7.54861 17.076 7.91994 17.1725 8.25764C17.1944 8.33437 17.2198 8.412 17.249 8.48921ZM16.6402 5.66864L9.34721 2.54307L3.61469 4.45391L11.8573 8.06006L16.6402 5.66864Z" fill="#000000" fill-opacity=".6" stroke="#000000" stroke-opacity=".6" stroke-width="0.3"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.0 KiB After Width: | Height: | Size: 3.0 KiB |
|
|
@ -97,6 +97,9 @@ const agentKBSelectionMode = computed(() => {
|
|||
return currentAgentConfig.value?.kb_selection_mode || 'all';
|
||||
});
|
||||
|
||||
// 共享智能体下的知识库列表(来自 listKnowledgeBases(agent_id)),用于已选知识库展示与 org 角标
|
||||
const sharedAgentKbList = ref<Array<{ id: string; name: string; type?: string; knowledge_count?: number; chunk_count?: number }>>([]);
|
||||
|
||||
// 当智能体改变时,模型、网络搜索、可@知识库列表均跟随新智能体配置
|
||||
// 知识库:用新智能体配置的列表替换当前选中,使已选与可@列表一致(含共享智能体)
|
||||
watch([selectedAgentId, agentKnowledgeBases, agentKBSelectionMode], ([newAgentId, newAgentKbs, newKbMode], [oldAgentId]) => {
|
||||
|
|
@ -205,9 +208,6 @@ const sharedAgentOrgName = computed(() => {
|
|||
return shared?.org_name || shared?.shared_by_username || '';
|
||||
});
|
||||
|
||||
// 共享智能体下的知识库列表(来自 listKnowledgeBases(agent_id)),用于已选知识库展示与 org 角标
|
||||
const sharedAgentKbList = ref<Array<{ id: string; name: string; type?: string; knowledge_count?: number; chunk_count?: number }>>([]);
|
||||
|
||||
const props = defineProps({
|
||||
isReplying: {
|
||||
type: Boolean,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { MessagePlugin, DialogPlugin } from "tdesign-vue-next";
|
|||
import { sanitizeHTML, safeMarkdownToHTML, createSafeImage, isValidImageURL } from '@/utils/security';
|
||||
import { openMermaidFullscreen } from '@/utils/mermaidViewer';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import DocumentPreview from '@/components/document-preview.vue';
|
||||
|
||||
const { t } = useI18n();
|
||||
|
||||
|
|
@ -57,10 +58,9 @@ let doc = null;
|
|||
let down = ref()
|
||||
let mdContentWrap = ref()
|
||||
let url = ref('')
|
||||
// 视图模式:chunks / original / merged
|
||||
const viewMode = ref<'chunks' | 'original' | 'merged'>('merged');
|
||||
const originalContent = ref<string>('');
|
||||
const loadingOriginal = ref(false);
|
||||
// 视图模式:chunks / merged / preview
|
||||
// file 类型默认「预览」,URL / 手动创建 默认「全文」
|
||||
const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged');
|
||||
|
||||
// 合并后的文档内容
|
||||
const mergedContent = ref<string>('');
|
||||
|
|
@ -212,6 +212,31 @@ watch(() => props.details?.md, (newChunks) => {
|
|||
}
|
||||
}, { immediate: true, deep: true });
|
||||
|
||||
const previewSupportedTypes = new Set([
|
||||
'pdf', 'docx', 'pptx', 'ppt', 'xlsx', 'xls', 'csv',
|
||||
'jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'tiff', 'svg',
|
||||
'txt', 'md', 'markdown', 'json', 'xml', 'html', 'css', 'js', 'ts',
|
||||
'py', 'java', 'go', 'cpp', 'c', 'h', 'sh', 'yaml', 'yml',
|
||||
'ini', 'conf', 'log', 'sql', 'rs', 'rb', 'php', 'swift', 'kt',
|
||||
'scala', 'r', 'lua', 'pl', 'toml',
|
||||
]);
|
||||
|
||||
const canPreview = (): boolean => {
|
||||
if (props.details?.type !== 'file') return false;
|
||||
const ft = props.details?.file_type?.toLowerCase();
|
||||
return !!ft && previewSupportedTypes.has(ft);
|
||||
};
|
||||
|
||||
// 当文档详情加载完成时,file 类型自动切换到「预览」
|
||||
watch(() => props.details?.id, (newId) => {
|
||||
if (!newId) return;
|
||||
if (props.details?.type === 'file' && canPreview()) {
|
||||
viewMode.value = 'preview';
|
||||
} else {
|
||||
viewMode.value = 'merged';
|
||||
}
|
||||
});
|
||||
|
||||
const isTextFile = (fileType?: string): boolean => {
|
||||
if (!fileType) return false;
|
||||
const textTypes = ['txt', 'md', 'markdown', 'json', 'xml', 'html', 'css', 'js', 'ts', 'py', 'java', 'go', 'cpp', 'c', 'h', 'sh', 'yaml', 'yml', 'ini', 'conf', 'log'];
|
||||
|
|
@ -222,25 +247,6 @@ const isMarkdownFile = (fileType?: string): boolean => {
|
|||
const markdownTypes = ['md', 'markdown'];
|
||||
return markdownTypes.includes(fileType.toLowerCase());
|
||||
};
|
||||
const loadOriginalContent = async () => {
|
||||
if (!props.details.id || !props.details.type || props.details.type !== 'file') return;
|
||||
const fileType = props.details.file_type?.toLowerCase();
|
||||
if (!isTextFile(fileType)) {
|
||||
MessagePlugin.warning(t('knowledgeBase.originalFileNotSupported') || '该文件类型不支持原文件展示,请下载查看');
|
||||
return;
|
||||
}
|
||||
loadingOriginal.value = true;
|
||||
try {
|
||||
const blob = await downKnowledgeDetails(props.details.id);
|
||||
const text = await blob.text();
|
||||
originalContent.value = text;
|
||||
} catch (error: any) {
|
||||
console.error('Failed to load original content:', error);
|
||||
MessagePlugin.error(error?.message || t('knowledgeBase.loadOriginalFailed') || '加载原文件内容失败');
|
||||
} finally {
|
||||
loadingOriginal.value = false;
|
||||
}
|
||||
};
|
||||
watch(() => props.details.md, (newVal) => {
|
||||
nextTick(async () => {
|
||||
const images = mdContentWrap.value.querySelectorAll('img.markdown-image');
|
||||
|
|
@ -346,7 +352,6 @@ const handleClose = () => {
|
|||
emit("closeDoc", false);
|
||||
doc.scrollTop = 0;
|
||||
viewMode.value = 'merged';
|
||||
originalContent.value = '';
|
||||
};
|
||||
|
||||
// 获取显示标题
|
||||
|
|
@ -622,6 +627,17 @@ const handleDetailsScroll = () => {
|
|||
<span class="time"> {{ getTimeLabel() }}:{{ details.time }} </span>
|
||||
<div class="view-mode-buttons">
|
||||
<t-button
|
||||
v-if="canPreview()"
|
||||
size="small"
|
||||
:variant="viewMode === 'preview' ? 'base' : 'outline'"
|
||||
:theme="viewMode === 'preview' ? 'primary' : 'default'"
|
||||
@click="viewMode = 'preview'"
|
||||
class="view-mode-btn"
|
||||
>
|
||||
{{ $t('preview.tab') || '预览' }}
|
||||
</t-button>
|
||||
<t-button
|
||||
v-if="!canPreview()"
|
||||
size="small"
|
||||
:variant="viewMode === 'merged' ? 'base' : 'outline'"
|
||||
:theme="viewMode === 'merged' ? 'primary' : 'default'"
|
||||
|
|
@ -639,7 +655,6 @@ const handleDetailsScroll = () => {
|
|||
>
|
||||
{{ $t('knowledgeBase.viewChunks') || '分块' }}
|
||||
</t-button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -709,6 +724,16 @@ const handleDetailsScroll = () => {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 文档预览视图 -->
|
||||
<div v-else-if="viewMode === 'preview'">
|
||||
<DocumentPreview
|
||||
:knowledgeId="details.id"
|
||||
:fileType="details.file_type"
|
||||
:fileName="details.title"
|
||||
:active="viewMode === 'preview'"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<template #footer>
|
||||
<t-button @click="handleClose">{{ $t('common.confirm') }}</t-button>
|
||||
<t-button theme="default" @click="handleClose">{{ $t('common.cancel') }}</t-button>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,625 @@
|
|||
// @ts-nocheck
|
||||
<script setup lang="ts">
|
||||
import { ref, shallowRef, watch, onUnmounted, nextTick, defineAsyncComponent } from 'vue';
|
||||
import { previewKnowledgeFile } from '@/api/knowledge-base/index';
|
||||
import { MessagePlugin } from 'tdesign-vue-next';
|
||||
import hljs from 'highlight.js';
|
||||
import 'highlight.js/styles/github.css';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
|
||||
const VueOfficePptx = defineAsyncComponent(() => import('@vue-office/pptx'));
|
||||
|
||||
const { t } = useI18n();
|
||||
|
||||
const props = defineProps<{
|
||||
knowledgeId: string;
|
||||
fileType: string;
|
||||
fileName: string;
|
||||
active: boolean;
|
||||
}>();
|
||||
|
||||
const loading = ref(false);
|
||||
const error = ref('');
|
||||
const previewType = ref<'pdf' | 'docx' | 'image' | 'excel' | 'text' | 'markdown' | 'pptx' | 'unsupported'>('unsupported');
|
||||
const blobUrl = ref('');
|
||||
const textContent = ref('');
|
||||
const highlightedCode = ref('');
|
||||
const markdownHtml = ref('');
|
||||
const excelHtml = ref('');
|
||||
const pptxData = shallowRef<ArrayBuffer | null>(null);
|
||||
const docxContainer = ref<HTMLElement | null>(null);
|
||||
const imageNaturalWidth = ref(0);
|
||||
const imageNaturalHeight = ref(0);
|
||||
let loadedForId = '';
|
||||
|
||||
const fileTypeMap: Record<string, typeof previewType.value> = {};
|
||||
['pdf'].forEach(t => fileTypeMap[t] = 'pdf');
|
||||
['docx'].forEach(t => fileTypeMap[t] = 'docx');
|
||||
['pptx', 'ppt'].forEach(t => fileTypeMap[t] = 'pptx');
|
||||
['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'tiff', 'svg'].forEach(t => fileTypeMap[t] = 'image');
|
||||
['xlsx', 'xls', 'csv'].forEach(t => fileTypeMap[t] = 'excel');
|
||||
['md', 'markdown'].forEach(t => fileTypeMap[t] = 'markdown');
|
||||
['txt', 'json', 'xml', 'html', 'css', 'js', 'ts', 'py', 'java', 'go',
|
||||
'cpp', 'c', 'h', 'sh', 'yaml', 'yml', 'ini', 'conf', 'log', 'sql', 'rs', 'rb', 'php',
|
||||
'swift', 'kt', 'scala', 'r', 'lua', 'pl', 'toml'].forEach(t => fileTypeMap[t] = 'text');
|
||||
|
||||
const mimeTypeMap: Record<string, string> = {
|
||||
pdf: 'application/pdf',
|
||||
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
doc: 'application/msword',
|
||||
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
ppt: 'application/vnd.ms-powerpoint',
|
||||
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
xls: 'application/vnd.ms-excel',
|
||||
csv: 'text/csv',
|
||||
jpg: 'image/jpeg', jpeg: 'image/jpeg',
|
||||
png: 'image/png', gif: 'image/gif', bmp: 'image/bmp',
|
||||
webp: 'image/webp', tiff: 'image/tiff', svg: 'image/svg+xml',
|
||||
txt: 'text/plain', md: 'text/markdown', markdown: 'text/markdown',
|
||||
json: 'application/json', xml: 'application/xml',
|
||||
html: 'text/html', css: 'text/css',
|
||||
js: 'text/javascript', ts: 'text/typescript',
|
||||
py: 'text/x-python', java: 'text/x-java', go: 'text/x-go',
|
||||
};
|
||||
|
||||
function getMimeType(ft: string): string {
|
||||
return mimeTypeMap[ft?.toLowerCase()] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
function ensureBlobType(blob: Blob, ft: string): Blob {
|
||||
const expected = getMimeType(ft);
|
||||
if (blob.type === expected) return blob;
|
||||
return new Blob([blob], { type: expected });
|
||||
}
|
||||
|
||||
const langMap: Record<string, string> = {
|
||||
js: 'javascript', ts: 'typescript', py: 'python', rb: 'ruby',
|
||||
sh: 'bash', yml: 'yaml', md: 'markdown', rs: 'rust',
|
||||
kt: 'kotlin', pl: 'perl', conf: 'ini', log: 'plaintext',
|
||||
};
|
||||
|
||||
function resolvePreviewType(ft: string): typeof previewType.value {
|
||||
return fileTypeMap[ft?.toLowerCase()] || 'unsupported';
|
||||
}
|
||||
|
||||
function getHighlightLang(ft: string): string {
|
||||
const lower = ft?.toLowerCase() || '';
|
||||
return langMap[lower] || lower;
|
||||
}
|
||||
|
||||
async function renderDocx(blob: Blob) {
|
||||
const { renderAsync } = await import('docx-preview');
|
||||
if (docxContainer.value) {
|
||||
docxContainer.value.innerHTML = '';
|
||||
await renderAsync(blob, docxContainer.value, undefined, {
|
||||
className: 'docx-preview-wrapper',
|
||||
inWrapper: true,
|
||||
ignoreWidth: false,
|
||||
ignoreHeight: false,
|
||||
ignoreFonts: false,
|
||||
breakPages: true,
|
||||
ignoreLastRenderedPageBreak: true,
|
||||
experimental: false,
|
||||
trimXmlDeclaration: true,
|
||||
useBase64URL: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function isValidUTF8(bytes: Uint8Array): boolean {
|
||||
for (let i = 0; i < bytes.length;) {
|
||||
const b = bytes[i];
|
||||
let remaining = 0;
|
||||
if (b <= 0x7F) { remaining = 0; }
|
||||
else if ((b & 0xE0) === 0xC0) { remaining = 1; }
|
||||
else if ((b & 0xF0) === 0xE0) { remaining = 2; }
|
||||
else if ((b & 0xF8) === 0xF0) { remaining = 3; }
|
||||
else { return false; }
|
||||
if (i + remaining >= bytes.length) return false;
|
||||
for (let j = 1; j <= remaining; j++) {
|
||||
if ((bytes[i + j] & 0xC0) !== 0x80) return false;
|
||||
}
|
||||
i += 1 + remaining;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function decodeCSVBlob(arrayBuffer: ArrayBuffer): string {
|
||||
const bytes = new Uint8Array(arrayBuffer);
|
||||
if (bytes[0] === 0xEF && bytes[1] === 0xBB && bytes[2] === 0xBF) {
|
||||
return new TextDecoder('utf-8').decode(bytes);
|
||||
}
|
||||
if (isValidUTF8(bytes)) {
|
||||
return new TextDecoder('utf-8').decode(bytes);
|
||||
}
|
||||
return new TextDecoder('gbk').decode(bytes);
|
||||
}
|
||||
|
||||
async function renderExcel(blob: Blob, fileType?: string) {
|
||||
const XLSX = await import('xlsx');
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
|
||||
let workbook;
|
||||
if (fileType?.toLowerCase() === 'csv') {
|
||||
const csvText = decodeCSVBlob(arrayBuffer);
|
||||
workbook = XLSX.read(csvText, { type: 'string' });
|
||||
} else {
|
||||
workbook = XLSX.read(arrayBuffer, { type: 'array' });
|
||||
}
|
||||
|
||||
let html = '';
|
||||
workbook.SheetNames.forEach((name, sheetIdx) => {
|
||||
const sheet = workbook.Sheets[name];
|
||||
const sheetHtml = XLSX.utils.sheet_to_html(sheet, { id: `sheet-${sheetIdx}` });
|
||||
html += `<div class="excel-sheet">`;
|
||||
if (workbook.SheetNames.length > 1) {
|
||||
html += `<div class="excel-sheet-name">${name}</div>`;
|
||||
}
|
||||
html += sheetHtml;
|
||||
html += `</div>`;
|
||||
});
|
||||
excelHtml.value = html;
|
||||
}
|
||||
|
||||
async function renderText(blob: Blob, fileType: string) {
|
||||
const text = await blob.text();
|
||||
textContent.value = text;
|
||||
|
||||
const lang = getHighlightLang(fileType);
|
||||
if (lang && hljs.getLanguage(lang)) {
|
||||
try {
|
||||
highlightedCode.value = hljs.highlight(text, { language: lang }).value;
|
||||
return;
|
||||
} catch { /* fallthrough */ }
|
||||
}
|
||||
const auto = hljs.highlightAuto(text);
|
||||
highlightedCode.value = auto.value;
|
||||
}
|
||||
|
||||
async function renderMarkdown(blob: Blob) {
|
||||
const { marked } = await import('marked');
|
||||
const text = await blob.text();
|
||||
marked.use({
|
||||
breaks: true,
|
||||
gfm: true,
|
||||
});
|
||||
const renderer = new marked.Renderer();
|
||||
renderer.code = function (code, infostring) {
|
||||
const lang = (infostring || '').trim();
|
||||
let highlighted = '';
|
||||
if (lang && hljs.getLanguage(lang)) {
|
||||
try { highlighted = hljs.highlight(code, { language: lang }).value; }
|
||||
catch { highlighted = hljs.highlightAuto(code).value; }
|
||||
} else {
|
||||
highlighted = hljs.highlightAuto(code).value;
|
||||
}
|
||||
return `<pre><code class="hljs">${highlighted}</code></pre>`;
|
||||
};
|
||||
marked.use({ renderer });
|
||||
markdownHtml.value = marked.parse(text);
|
||||
}
|
||||
|
||||
function onImageLoad(e: Event) {
|
||||
const img = e.target as HTMLImageElement;
|
||||
imageNaturalWidth.value = img.naturalWidth;
|
||||
imageNaturalHeight.value = img.naturalHeight;
|
||||
}
|
||||
|
||||
async function loadPreview() {
|
||||
const id = props.knowledgeId;
|
||||
const ft = props.fileType;
|
||||
if (!id || !ft) return;
|
||||
if (loadedForId === id) return;
|
||||
|
||||
cleanup();
|
||||
loading.value = true;
|
||||
error.value = '';
|
||||
previewType.value = resolvePreviewType(ft);
|
||||
|
||||
if (previewType.value === 'unsupported') {
|
||||
loading.value = false;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const rawBlob = await previewKnowledgeFile(id);
|
||||
const blob = ensureBlobType(rawBlob, ft);
|
||||
loadedForId = id;
|
||||
|
||||
loading.value = false;
|
||||
await nextTick();
|
||||
|
||||
switch (previewType.value) {
|
||||
case 'pdf': {
|
||||
blobUrl.value = URL.createObjectURL(blob);
|
||||
break;
|
||||
}
|
||||
case 'image': {
|
||||
blobUrl.value = URL.createObjectURL(blob);
|
||||
break;
|
||||
}
|
||||
case 'docx': {
|
||||
await renderDocx(blob);
|
||||
break;
|
||||
}
|
||||
case 'excel': {
|
||||
await renderExcel(blob, ft);
|
||||
break;
|
||||
}
|
||||
case 'text': {
|
||||
await renderText(blob, ft);
|
||||
break;
|
||||
}
|
||||
case 'markdown': {
|
||||
await renderMarkdown(blob);
|
||||
break;
|
||||
}
|
||||
case 'pptx': {
|
||||
pptxData.value = await blob.arrayBuffer();
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error('Document preview failed:', err);
|
||||
error.value = err?.message || t('preview.loadFailed');
|
||||
} finally {
|
||||
loading.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
if (blobUrl.value) {
|
||||
URL.revokeObjectURL(blobUrl.value);
|
||||
blobUrl.value = '';
|
||||
}
|
||||
textContent.value = '';
|
||||
highlightedCode.value = '';
|
||||
markdownHtml.value = '';
|
||||
excelHtml.value = '';
|
||||
pptxData.value = null;
|
||||
imageNaturalWidth.value = 0;
|
||||
imageNaturalHeight.value = 0;
|
||||
loadedForId = '';
|
||||
if (docxContainer.value) {
|
||||
docxContainer.value.innerHTML = '';
|
||||
}
|
||||
}
|
||||
|
||||
watch(
|
||||
() => [props.active, props.knowledgeId],
|
||||
([active]) => {
|
||||
if (active && props.knowledgeId) {
|
||||
loadPreview();
|
||||
}
|
||||
},
|
||||
{ immediate: true }
|
||||
);
|
||||
|
||||
onUnmounted(() => {
|
||||
cleanup();
|
||||
});
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="document-preview">
|
||||
<!-- Loading -->
|
||||
<div v-if="loading" class="preview-loading">
|
||||
<t-loading size="large" />
|
||||
<span class="loading-text">{{ $t('preview.loading') }}</span>
|
||||
</div>
|
||||
|
||||
<!-- Error -->
|
||||
<div v-else-if="error" class="preview-error">
|
||||
<t-icon name="error-circle" size="48px" />
|
||||
<p>{{ error }}</p>
|
||||
<t-button theme="primary" size="small" @click="loadedForId = ''; loadPreview()">
|
||||
{{ $t('preview.retry') }}
|
||||
</t-button>
|
||||
</div>
|
||||
|
||||
<!-- Unsupported -->
|
||||
<div v-else-if="previewType === 'unsupported'" class="preview-unsupported">
|
||||
<t-icon name="file-unknown" size="48px" />
|
||||
<p>{{ $t('preview.unsupported') }}</p>
|
||||
<p class="unsupported-hint">{{ $t('preview.unsupportedHint') }}</p>
|
||||
</div>
|
||||
|
||||
<!-- PDF -->
|
||||
<div v-else-if="previewType === 'pdf' && blobUrl" class="preview-pdf">
|
||||
<iframe :src="blobUrl" class="pdf-iframe" />
|
||||
</div>
|
||||
|
||||
<!-- Image -->
|
||||
<div v-else-if="previewType === 'image' && blobUrl" class="preview-image">
|
||||
<div class="image-wrapper">
|
||||
<img :src="blobUrl" :alt="fileName" @load="onImageLoad" />
|
||||
<div v-if="imageNaturalWidth" class="image-info">
|
||||
{{ imageNaturalWidth }} × {{ imageNaturalHeight }} px
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- DOCX -->
|
||||
<div v-else-if="previewType === 'docx'" class="preview-docx">
|
||||
<div ref="docxContainer" class="docx-container" />
|
||||
</div>
|
||||
|
||||
<!-- PPTX -->
|
||||
<div v-else-if="previewType === 'pptx' && pptxData" class="preview-pptx">
|
||||
<vue-office-pptx :src="pptxData" @rendered="() => {}" @error="(e: any) => { error = e?.message || $t('preview.loadFailed'); }" />
|
||||
</div>
|
||||
|
||||
<!-- Excel -->
|
||||
<div v-else-if="previewType === 'excel' && excelHtml" class="preview-excel">
|
||||
<div class="excel-container" v-html="excelHtml" />
|
||||
</div>
|
||||
|
||||
<!-- Markdown -->
|
||||
<div v-else-if="previewType === 'markdown' && markdownHtml" class="preview-markdown">
|
||||
<div class="markdown-body" v-html="markdownHtml" />
|
||||
</div>
|
||||
|
||||
<!-- Text / Code -->
|
||||
<div v-else-if="previewType === 'text' && highlightedCode" class="preview-text">
|
||||
<pre class="code-preview"><code class="hljs" v-html="highlightedCode"></code></pre>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="less">
|
||||
// ── Design tokens ──
|
||||
@border-color: #e7e7e7;
|
||||
@border-radius: 6px;
|
||||
@bg-white: #fff;
|
||||
@bg-subtle: #f6f8fa;
|
||||
@bg-muted: #f9fafb;
|
||||
@text-primary: #1d2129;
|
||||
@text-secondary: #000000a6;
|
||||
@text-tertiary: #00000066;
|
||||
@text-disabled: #00000040;
|
||||
@accent: #07c05f;
|
||||
@accent-hover: #059669;
|
||||
@accent-bg: #f0fdf4;
|
||||
@accent-bg-hover: #e6f7ed;
|
||||
@error-color: #e34d59;
|
||||
@table-border: #e7e7e7;
|
||||
@preview-max-h: calc(100vh - 200px);
|
||||
@transition: all 0.2s ease;
|
||||
|
||||
// ── Shared container mixin ──
|
||||
.preview-container() {
|
||||
border: 1px solid @border-color;
|
||||
border-radius: @border-radius;
|
||||
overflow: auto;
|
||||
max-height: @preview-max-h;
|
||||
background: @bg-white;
|
||||
}
|
||||
|
||||
.document-preview {
|
||||
min-height: 200px;
|
||||
}
|
||||
|
||||
// ── States ──
|
||||
.preview-loading {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 60px 20px;
|
||||
gap: 16px;
|
||||
.loading-text { color: @text-tertiary; font-size: 14px; }
|
||||
}
|
||||
|
||||
.preview-error {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 60px 20px;
|
||||
gap: 12px;
|
||||
color: @error-color;
|
||||
p { margin: 0; font-size: 14px; color: @text-secondary; }
|
||||
}
|
||||
|
||||
.preview-unsupported {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 60px 20px;
|
||||
gap: 12px;
|
||||
color: @text-disabled;
|
||||
p { margin: 0; font-size: 14px; color: @text-secondary; }
|
||||
.unsupported-hint { font-size: 12px; color: @text-tertiary; }
|
||||
}
|
||||
|
||||
// ── PDF ──
|
||||
.preview-pdf {
|
||||
width: 100%;
|
||||
height: @preview-max-h;
|
||||
min-height: 500px;
|
||||
.pdf-iframe {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: none;
|
||||
border-radius: @border-radius;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Image ──
|
||||
.preview-image {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 20px 0;
|
||||
.image-wrapper {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
img {
|
||||
max-width: 100%;
|
||||
max-height: calc(100vh - 280px);
|
||||
border-radius: @border-radius;
|
||||
box-shadow: 0 2px 12px rgba(7, 192, 95, 0.08);
|
||||
object-fit: contain;
|
||||
}
|
||||
.image-info { font-size: 12px; color: @text-tertiary; }
|
||||
}
|
||||
}
|
||||
|
||||
// ── Markdown ──
|
||||
.preview-markdown {
|
||||
.preview-container();
|
||||
padding: 20px 24px;
|
||||
}
|
||||
|
||||
// ── DOCX ──
|
||||
.preview-docx {
|
||||
.docx-container { .preview-container(); }
|
||||
}
|
||||
|
||||
// ── PPTX ──
|
||||
.preview-pptx {
|
||||
height: @preview-max-h;
|
||||
min-height: 500px;
|
||||
border: 1px solid @border-color;
|
||||
border-radius: @border-radius;
|
||||
overflow: hidden;
|
||||
background: @bg-subtle;
|
||||
}
|
||||
|
||||
// ── Excel ──
|
||||
.preview-excel {
|
||||
.excel-container { .preview-container(); }
|
||||
}
|
||||
|
||||
// ── Text / Code ──
|
||||
.preview-text {
|
||||
.code-preview {
|
||||
.preview-container();
|
||||
margin: 0;
|
||||
padding: 16px;
|
||||
background: @bg-subtle;
|
||||
font-size: 13px;
|
||||
line-height: 1.6;
|
||||
code {
|
||||
white-space: pre;
|
||||
word-wrap: normal;
|
||||
display: block;
|
||||
background: transparent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Deep styles (v-html / third-party components) ──
|
||||
|
||||
// Shared table mixin for v-html content
|
||||
.preview-table() {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 13px;
|
||||
th, td {
|
||||
border: 1px solid @table-border;
|
||||
padding: 6px 12px;
|
||||
text-align: left;
|
||||
}
|
||||
th {
|
||||
background: @accent-bg;
|
||||
font-weight: 600;
|
||||
color: @text-primary;
|
||||
}
|
||||
tr:hover td {
|
||||
background: @accent-bg;
|
||||
transition: @transition;
|
||||
}
|
||||
}
|
||||
|
||||
:deep(.markdown-body) {
|
||||
font-size: 14px;
|
||||
line-height: 1.7;
|
||||
color: @text-primary;
|
||||
word-break: break-word;
|
||||
|
||||
h1, h2, h3, h4, h5, h6 {
|
||||
margin-top: 20px;
|
||||
margin-bottom: 10px;
|
||||
font-weight: 600;
|
||||
line-height: 1.4;
|
||||
}
|
||||
h1 { font-size: 24px; border-bottom: 1px solid @border-color; padding-bottom: 8px; }
|
||||
h2 { font-size: 20px; border-bottom: 1px solid @border-color; padding-bottom: 6px; }
|
||||
h3 { font-size: 17px; }
|
||||
|
||||
p { margin: 8px 0; }
|
||||
blockquote {
|
||||
margin: 12px 0;
|
||||
padding: 8px 16px;
|
||||
border-left: 4px solid @accent;
|
||||
background: @bg-subtle;
|
||||
color: #57606a;
|
||||
}
|
||||
ul, ol { padding-left: 24px; margin: 8px 0; }
|
||||
li { margin: 4px 0; }
|
||||
|
||||
table { .preview-table(); margin: 12px 0; }
|
||||
|
||||
pre {
|
||||
margin: 12px 0;
|
||||
padding: 14px;
|
||||
background: @bg-subtle;
|
||||
border-radius: @border-radius;
|
||||
overflow: auto;
|
||||
font-size: 13px;
|
||||
line-height: 1.5;
|
||||
code { background: transparent; padding: 0; }
|
||||
}
|
||||
code {
|
||||
background: #eff1f3;
|
||||
padding: 2px 6px;
|
||||
border-radius: 3px;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
img { max-width: 100%; border-radius: 4px; }
|
||||
hr { border: none; border-top: 1px solid @border-color; margin: 20px 0; }
|
||||
a { color: @accent; text-decoration: none; &:hover { color: @accent-hover; text-decoration: underline; } }
|
||||
strong { font-weight: 600; }
|
||||
}
|
||||
|
||||
:deep(.docx-preview-wrapper) {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
:deep(.vue-office-pptx) {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
:deep(.vue-office-pptx-main) {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
:deep(.excel-sheet) {
|
||||
padding: 0;
|
||||
.excel-sheet-name {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: @accent-bg;
|
||||
padding: 8px 16px;
|
||||
font-weight: 600;
|
||||
font-size: 13px;
|
||||
color: @text-primary;
|
||||
border-bottom: 1px solid @border-color;
|
||||
z-index: 1;
|
||||
}
|
||||
table {
|
||||
.preview-table();
|
||||
th, td {
|
||||
white-space: nowrap;
|
||||
max-width: 300px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
<div class="aside_box">
|
||||
<div class="logo_box" @click="router.push('/platform/knowledge-bases')" style="cursor: pointer;">
|
||||
<img class="logo" src="@/assets/img/weknora.png" alt="">
|
||||
<span v-if="isLiteEdition" class="lite-badge">Lite</span>
|
||||
</div>
|
||||
|
||||
<!-- 租户选择器:仅在用户可切换租户时显示 -->
|
||||
|
|
@ -80,6 +81,7 @@ import UserMenu from '@/components/UserMenu.vue';
|
|||
import TenantSelector from '@/components/TenantSelector.vue';
|
||||
import BatchManageDialog from '@/components/BatchManageDialog.vue';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { getSystemInfo } from '@/api/system';
|
||||
|
||||
const { t } = useI18n();
|
||||
const usemenuStore = useMenuStore();
|
||||
|
|
@ -100,6 +102,7 @@ const hasMore = computed(() => currentPage.value < totalPages.value);
|
|||
type MenuItem = { title: string; icon: string; path: string; childrenPath?: string; children?: any[] };
|
||||
const { menuArr } = storeToRefs(usemenuStore);
|
||||
let activeSubmenu = ref<string>('');
|
||||
const isLiteEdition = ref(false);
|
||||
|
||||
// 批量管理状态
|
||||
const batchManageVisible = ref(false);
|
||||
|
|
@ -380,6 +383,10 @@ onMounted(async () => {
|
|||
if (route.params.chatid) {
|
||||
currentSecondpath.value = `chat/${route.params.chatid}`;
|
||||
}
|
||||
|
||||
getSystemInfo().then(res => {
|
||||
isLiteEdition.value = res.data?.edition === 'lite'
|
||||
}).catch(() => {})
|
||||
|
||||
// 初始化知识库信息
|
||||
const kbId = (route.params as any)?.kbId as string
|
||||
|
|
@ -593,6 +600,19 @@ const mouseleaveMenu = (path: string) => {
|
|||
height: auto;
|
||||
margin-left: 24px;
|
||||
}
|
||||
.lite-badge {
|
||||
margin-left: 4px;
|
||||
padding: 1px 6px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
line-height: 18px;
|
||||
color: #07c05f;
|
||||
background: #07c05f1a;
|
||||
border: 1px solid #07c05f40;
|
||||
border-radius: 4px;
|
||||
letter-spacing: 0.5px;
|
||||
user-select: none;
|
||||
}
|
||||
}
|
||||
|
||||
.logo_img {
|
||||
|
|
@ -699,6 +719,8 @@ const mouseleaveMenu = (path: string) => {
|
|||
padding: 13px 8px 13px 16px;
|
||||
box-sizing: border-box;
|
||||
margin-bottom: 4px;
|
||||
border-radius: 4px;
|
||||
transition: background-color 0.2s ease;
|
||||
|
||||
.menu_item-box {
|
||||
display: flex;
|
||||
|
|
@ -707,12 +729,11 @@ const mouseleaveMenu = (path: string) => {
|
|||
|
||||
&:hover {
|
||||
border-radius: 4px;
|
||||
background: #30323605;
|
||||
color: #00000099;
|
||||
background: rgba(0, 0, 0, 0.04);
|
||||
|
||||
.menu_icon,
|
||||
.menu_title {
|
||||
color: #00000099;
|
||||
color: #000000b3;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -812,12 +833,12 @@ const mouseleaveMenu = (path: string) => {
|
|||
}
|
||||
|
||||
&:hover {
|
||||
background: #30323605;
|
||||
color: #00000099;
|
||||
background: rgba(0, 0, 0, 0.04);
|
||||
color: #000000b3;
|
||||
border-radius: 3px;
|
||||
|
||||
.menu-more {
|
||||
color: #00000099;
|
||||
color: #000000b3;
|
||||
}
|
||||
|
||||
.menu-more-wrap {
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ export default {
|
|||
imagesFilteredNoVLM: 'Filtered {count} image files (VLM not enabled)',
|
||||
invalidFilesFiltered: 'Filtered {count} unsupported files',
|
||||
unsupportedFileType: 'Unsupported file type',
|
||||
unsupportedTypesHint: 'Some document types ({types}) have no available parser engine and cannot be processed',
|
||||
goToParserSettings: 'Configure',
|
||||
failedFilesList: 'Failed files:',
|
||||
andMoreFiles: '...and {count} more files',
|
||||
duplicateFilesSkipped: '{count} duplicate files skipped',
|
||||
|
|
@ -504,6 +506,7 @@ export default {
|
|||
webSearchConfig: 'Web Search',
|
||||
enableMemory: 'Enable Memory',
|
||||
enableMemoryDesc: 'When enabled, the system will record your conversation history and automatically recall relevant content in future conversations to provide more personalized answers.',
|
||||
storageEngine: 'Storage Engine',
|
||||
mcpService: 'MCP Service',
|
||||
systemSettings: 'System Settings',
|
||||
tenantInfo: 'Tenant Info',
|
||||
|
|
@ -1216,6 +1219,7 @@ export default {
|
|||
basic: 'Basic Information',
|
||||
models: 'Model Configuration',
|
||||
chunking: 'Chunking Settings',
|
||||
storage: 'Storage Engine',
|
||||
advanced: 'Advanced Settings',
|
||||
faq: 'FAQ Settings',
|
||||
graph: 'Knowledge Graph',
|
||||
|
|
@ -1503,6 +1507,11 @@ export default {
|
|||
deepThoughtCompleted: 'Deep thinking completed',
|
||||
deepThoughtAlt: 'Deep thinking finished',
|
||||
referencesTitle: 'Referenced {count} related item(s)',
|
||||
referencesDocCount: 'Referenced {count} document(s)',
|
||||
referencesDocAndWebCount: 'Referenced {docCount} document(s) and {webCount} web page(s)',
|
||||
referenceChunkCount: '{count} chunk(s)',
|
||||
chunkLabel: 'Chunk {index}:',
|
||||
navigateToDocument: 'View document details',
|
||||
referenceIconAlt: 'Reference materials icon',
|
||||
chunkIdLabel: 'Chunk ID:',
|
||||
documentIdLabel: 'Document ID:',
|
||||
|
|
@ -2698,4 +2707,12 @@ export default {
|
|||
sharedAgents: ' agents',
|
||||
},
|
||||
},
|
||||
preview: {
|
||||
tab: 'Preview',
|
||||
loading: 'Loading document preview...',
|
||||
loadFailed: 'Failed to load document preview',
|
||||
retry: 'Retry',
|
||||
unsupported: 'This file type does not support online preview',
|
||||
unsupportedHint: 'Please download and open with a local application',
|
||||
},
|
||||
}
|
||||
|
|
@ -96,6 +96,8 @@ export default {
|
|||
imagesFilteredNoVLM: "{count}개 이미지 파일 필터링됨(VLM 미활성화)",
|
||||
invalidFilesFiltered: "{count}개 지원되지 않는 파일 필터링됨",
|
||||
unsupportedFileType: "지원되지 않는 파일 형식",
|
||||
unsupportedTypesHint: "일부 문서 유형({types})에 사용 가능한 파서 엔진이 없어 처리할 수 없습니다",
|
||||
goToParserSettings: "설정으로 이동",
|
||||
failedFilesList: "실패한 파일 목록:",
|
||||
andMoreFiles: "...외 {count}개 파일",
|
||||
duplicateFilesSkipped: "{count}개 중복 파일 무시됨",
|
||||
|
|
@ -273,6 +275,11 @@ export default {
|
|||
deepThoughtCompleted: "심층 분석 완료",
|
||||
deepThoughtAlt: "심층 분석 완료",
|
||||
referencesTitle: "{count}개의 관련 내용 참조",
|
||||
referencesDocCount: "{count}개 문서 참조",
|
||||
referencesDocAndWebCount: "{docCount}개 문서와 {webCount}개 웹페이지 참조",
|
||||
referenceChunkCount: "{count}개 청크",
|
||||
chunkLabel: "청크 {index}:",
|
||||
navigateToDocument: "문서 상세 보기",
|
||||
referenceIconAlt: "참조 내용 아이콘",
|
||||
chunkIdLabel: "청크 ID:",
|
||||
documentIdLabel: "문서 ID:",
|
||||
|
|
@ -2777,4 +2784,12 @@ export default {
|
|||
sharedAgents: "에이전트",
|
||||
},
|
||||
},
|
||||
preview: {
|
||||
tab: '미리보기',
|
||||
loading: '문서 미리보기 로딩 중...',
|
||||
loadFailed: '문서 미리보기 로드 실패',
|
||||
retry: '재시도',
|
||||
unsupported: '이 파일 유형은 온라인 미리보기를 지원하지 않습니다',
|
||||
unsupportedHint: '파일을 다운로드하여 로컬 앱으로 열어주세요',
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -77,6 +77,8 @@ export default {
|
|||
imagesFilteredNoVLM: 'Отфильтровано {count} изображений (VLM не включен)',
|
||||
invalidFilesFiltered: 'Отфильтровано {count} неподдерживаемых файлов',
|
||||
unsupportedFileType: 'Неподдерживаемый тип файла',
|
||||
unsupportedTypesHint: 'Некоторые типы документов ({types}) не имеют доступного парсера и не могут быть обработаны',
|
||||
goToParserSettings: 'Настроить',
|
||||
failedFilesList: 'Неудавшиеся файлы:',
|
||||
andMoreFiles: '...и ещё {count} файлов',
|
||||
duplicateFilesSkipped: 'Пропущено {count} повторяющихся файлов',
|
||||
|
|
@ -1331,6 +1333,11 @@ export default {
|
|||
unableToGetKnowledgeBaseId: 'Невозможно получить ID базы знаний',
|
||||
summaryInProgress: 'Идёт подготовка ответа…',
|
||||
referencesTitle: 'Использовано {count} связанного материала',
|
||||
referencesDocCount: 'Использовано {count} документ(ов)',
|
||||
referencesDocAndWebCount: 'Использовано {docCount} документ(ов) и {webCount} веб-страниц(ы)',
|
||||
referenceChunkCount: '{count} фрагмент(ов)',
|
||||
chunkLabel: 'Фрагмент {index}:',
|
||||
navigateToDocument: 'Просмотр документа',
|
||||
referenceIconAlt: 'Иконка ссылок на материалы',
|
||||
chunkIdLabel: 'ID фрагмента:',
|
||||
documentIdLabel: 'ID документа:',
|
||||
|
|
@ -1841,5 +1848,13 @@ export default {
|
|||
stopSuccess: 'Генерация остановлена',
|
||||
stopFailed: 'Не удалось остановить. Попробуйте ещё раз.'
|
||||
}
|
||||
}
|
||||
},
|
||||
preview: {
|
||||
tab: 'Предпросмотр',
|
||||
loading: 'Загрузка предпросмотра документа...',
|
||||
loadFailed: 'Не удалось загрузить предпросмотр документа',
|
||||
retry: 'Повторить',
|
||||
unsupported: 'Этот тип файла не поддерживает онлайн-просмотр',
|
||||
unsupportedHint: 'Скачайте файл и откройте локально',
|
||||
},
|
||||
}
|
||||
|
|
@ -96,6 +96,8 @@ export default {
|
|||
imagesFilteredNoVLM: "已过滤 {count} 个图片文件(未启用VLM)",
|
||||
invalidFilesFiltered: "已过滤 {count} 个不支持的文件",
|
||||
unsupportedFileType: "不支持的文件格式",
|
||||
unsupportedTypesHint: "部分文档类型({types})暂无可用解析引擎,上传后将无法解析",
|
||||
goToParserSettings: "前往配置",
|
||||
failedFilesList: "失败文件列表:",
|
||||
andMoreFiles: "...及其他 {count} 个文件",
|
||||
duplicateFilesSkipped: "已忽略 {count} 个重复文件",
|
||||
|
|
@ -270,6 +272,11 @@ export default {
|
|||
deepThoughtCompleted: "已深度思考",
|
||||
deepThoughtAlt: "深度思考完成",
|
||||
referencesTitle: "参考了{count}个相关内容",
|
||||
referencesDocCount: "引用了{count}篇文档",
|
||||
referencesDocAndWebCount: "引用了{docCount}篇文档和{webCount}条网页",
|
||||
referenceChunkCount: "{count}个片段",
|
||||
chunkLabel: "片段{index}:",
|
||||
navigateToDocument: "查看文档详情",
|
||||
referenceIconAlt: "参考内容图标",
|
||||
chunkIdLabel: "片段ID:",
|
||||
documentIdLabel: "文档ID:",
|
||||
|
|
@ -346,6 +353,7 @@ export default {
|
|||
webSearchConfig: "网络搜索",
|
||||
enableMemory: "开启记忆功能",
|
||||
enableMemoryDesc: "开启后,系统将记录您的对话历史,并在后续对话中自动回忆相关内容,提供更个性化的回答。",
|
||||
storageEngine: "存储引擎",
|
||||
mcpService: "MCP服务",
|
||||
systemSettings: "系统设置",
|
||||
tenantInfo: "租户信息",
|
||||
|
|
@ -1612,6 +1620,7 @@ export default {
|
|||
basic: "基本信息",
|
||||
models: "模型配置",
|
||||
chunking: "分块设置",
|
||||
storage: "存储引擎",
|
||||
advanced: "高级设置",
|
||||
faq: "FAQ 设置",
|
||||
graph: "知识图谱",
|
||||
|
|
@ -2703,4 +2712,12 @@ export default {
|
|||
sharedAgents: "个智能体",
|
||||
},
|
||||
},
|
||||
preview: {
|
||||
tab: "预览",
|
||||
loading: "正在加载文档预览...",
|
||||
loadFailed: "加载文档预览失败",
|
||||
retry: "重试",
|
||||
unsupported: "该文件类型暂不支持在线预览",
|
||||
unsupportedHint: "请下载文件后使用本地应用查看",
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -39,29 +39,29 @@ export function formatStringDate(date: any) {
|
|||
year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second
|
||||
);
|
||||
}
|
||||
export function kbFileTypeVerification(file: any, silent = false) {
|
||||
let validTypes = ["pdf", "txt", "md", "docx", "doc", "jpg", "jpeg", "png", "csv", "xlsx", "xls"];
|
||||
let type = file.name.substring(file.name.lastIndexOf(".") + 1);
|
||||
if (!validTypes.includes(type)) {
|
||||
const DEFAULT_VALID_TYPES = new Set(["pdf", "txt", "md", "docx", "doc", "pptx", "ppt", "jpg", "jpeg", "png", "csv", "xlsx", "xls"]);
|
||||
|
||||
/**
|
||||
* Returns true when the file should be **rejected**.
|
||||
* @param validTypes - override the default extension whitelist with a dynamic set (e.g. from engine registry).
|
||||
*/
|
||||
export function kbFileTypeVerification(file: any, silent = false, validTypes?: Set<string> | string[]) {
|
||||
const allowed = validTypes
|
||||
? (validTypes instanceof Set ? validTypes : new Set(validTypes))
|
||||
: DEFAULT_VALID_TYPES;
|
||||
|
||||
const type = file.name.substring(file.name.lastIndexOf(".") + 1).toLowerCase();
|
||||
if (!allowed.has(type)) {
|
||||
if (!silent) {
|
||||
MessagePlugin.error("文件类型错误!");
|
||||
MessagePlugin.error("不支持的文件类型!");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
(type == "pdf" || type == "docx" || type == "doc") &&
|
||||
file.size > MAX_FILE_SIZE_BYTES
|
||||
) {
|
||||
if (file.size > MAX_FILE_SIZE_BYTES) {
|
||||
if (!silent) {
|
||||
MessagePlugin.error(`pdf/doc文件不能超过${MAX_FILE_SIZE_MB}M!`);
|
||||
MessagePlugin.error(`文件大小不能超过 ${MAX_FILE_SIZE_MB}M!`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if ((type == "txt" || type == "md") && file.size > MAX_FILE_SIZE_BYTES) {
|
||||
if (!silent) {
|
||||
MessagePlugin.error(`txt/md文件不能超过${MAX_FILE_SIZE_MB}M!`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -140,10 +140,14 @@ export function isValidURL(url: string): boolean {
|
|||
if (!url || typeof url !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 允许以 / 开头的站内相对路径(如本地存储 /files/images/xxx.jpg)
|
||||
if (url.startsWith('/') && !url.startsWith('//')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
// 只允许 http 和 https 协议
|
||||
return ['http:', 'https:'].includes(urlObj.protocol);
|
||||
} catch {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -890,7 +890,7 @@
|
|||
</div>
|
||||
<div class="setting-control">
|
||||
<div class="slider-wrapper">
|
||||
<t-slider v-model="formData.config.keyword_threshold" :min="0" :max="1" :step="0.05" />
|
||||
<t-slider v-model="formData.config.keyword_threshold" :min="0" :max="1" :step="0.01" />
|
||||
<span class="slider-value">{{ formData.config.keyword_threshold?.toFixed(2) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -904,7 +904,7 @@
|
|||
</div>
|
||||
<div class="setting-control">
|
||||
<div class="slider-wrapper">
|
||||
<t-slider v-model="formData.config.vector_threshold" :min="0" :max="1" :step="0.05" />
|
||||
<t-slider v-model="formData.config.vector_threshold" :min="0" :max="1" :step="0.01" />
|
||||
<span class="slider-value">{{ formData.config.vector_threshold?.toFixed(2) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -929,7 +929,7 @@
|
|||
</div>
|
||||
<div class="setting-control">
|
||||
<div class="slider-wrapper">
|
||||
<t-slider v-model="formData.config.rerank_threshold" :min="0" :max="1" :step="0.05" />
|
||||
<t-slider v-model="formData.config.rerank_threshold" :min="0" :max="1" :step="0.01" />
|
||||
<span class="slider-value">{{ formData.config.rerank_threshold?.toFixed(2) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -3,48 +3,71 @@
|
|||
<div class="refer_header" @click="referBoxSwitch" v-if="session.knowledge_references && session.knowledge_references.length">
|
||||
<div class="refer_title">
|
||||
<img src="@/assets/img/ziliao.svg" :alt="$t('chat.referenceIconAlt')" />
|
||||
<span>{{ $t('chat.referencesTitle', { count: session.knowledge_references?.length ?? 0 }) }}</span>
|
||||
<span>{{ headerText }}</span>
|
||||
</div>
|
||||
<div class="refer_show_icon">
|
||||
<t-icon :name="showReferBox ? 'chevron-up' : 'chevron-down'" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="refer_box" v-show="showReferBox">
|
||||
<div v-for="(item, index) in session.knowledge_references" :key="index">
|
||||
<!-- Web search references: show URL and make it clickable -->
|
||||
<template v-if="item.chunk_type === 'web_search'">
|
||||
<a
|
||||
:href="getWebSearchUrl(item)"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="doc doc-web"
|
||||
@click.stop
|
||||
>
|
||||
{{ session.knowledge_references.length < 2 ? getWebSearchDisplayText(item) : `${index + 1}.${getWebSearchDisplayText(item)}` }}
|
||||
</a>
|
||||
</template>
|
||||
<!-- Regular knowledge references: show title with popup -->
|
||||
<template v-else>
|
||||
<t-popup overlayClassName="refer-to-layer" placement="bottom-left" width="400" :showArrow="false"
|
||||
trigger="click">
|
||||
<template #content>
|
||||
<ContentPopup :content="safeProcessContent(item.content)" :is-html="true" />
|
||||
</template>
|
||||
<span class="doc">
|
||||
{{ session.knowledge_references.length < 2 ? item.knowledge_title : `${index +
|
||||
1}.${item.knowledge_title}` }} </span>
|
||||
</t-popup>
|
||||
</template>
|
||||
<!-- Web search references (ungrouped) -->
|
||||
<div v-for="(item, index) in webSearchRefs" :key="'web-' + index">
|
||||
<a
|
||||
:href="getWebSearchUrl(item)"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="doc doc-web"
|
||||
@click.stop
|
||||
>
|
||||
{{ webSearchRefs.length < 2 ? getWebSearchDisplayText(item) : `${index + 1}. ${getWebSearchDisplayText(item)}` }}
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- Knowledge references grouped by document -->
|
||||
<div v-for="(group, gIdx) in groupedKnowledgeRefs" :key="'grp-' + gIdx" class="doc-group">
|
||||
<div class="doc-group-header" @click="toggleGroup(group.key)">
|
||||
<div class="doc-group-left">
|
||||
<t-icon :name="expandedGroups[group.key] ? 'chevron-down' : 'chevron-right'" size="14px" class="doc-group-arrow" />
|
||||
<t-icon name="file" size="14px" class="doc-group-icon" />
|
||||
<span class="doc-group-title" :title="group.title">{{ group.title }}</span>
|
||||
<span class="doc-group-count">{{ $t('chat.referenceChunkCount', { count: group.chunks.length }) }}</span>
|
||||
</div>
|
||||
<div class="doc-group-actions" v-if="group.knowledgeBaseId" @click.stop>
|
||||
<t-tooltip :content="$t('chat.navigateToDocument')">
|
||||
<span class="doc-group-navigate" @click="navigateToDocument(group)">
|
||||
<t-icon name="jump" size="14px" />
|
||||
</span>
|
||||
</t-tooltip>
|
||||
</div>
|
||||
</div>
|
||||
<div class="doc-group-chunks" v-show="expandedGroups[group.key]">
|
||||
<div v-for="(chunk, cIdx) in group.chunks" :key="'chunk-' + cIdx" class="doc-chunk-item">
|
||||
<t-popup overlayClassName="refer-to-layer" placement="bottom-left" width="400" :showArrow="false" trigger="click">
|
||||
<template #content>
|
||||
<ContentPopup :content="safeProcessContent(chunk.content)" :is-html="true" />
|
||||
</template>
|
||||
<span class="doc-chunk-text">
|
||||
<span class="doc-chunk-index">{{ $t('chat.chunkLabel', { index: cIdx + 1 }) }}</span>
|
||||
{{ truncateContent(chunk.content, 80) }}
|
||||
</span>
|
||||
</t-popup>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
import { onMounted, defineProps, computed, ref, reactive } from "vue";
|
||||
import { defineProps, computed, ref, reactive } from "vue";
|
||||
import { useRouter } from 'vue-router';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { sanitizeHTML } from '@/utils/security';
|
||||
import ContentPopup from './tool-results/ContentPopup.vue';
|
||||
|
||||
const router = useRouter();
|
||||
const { t } = useI18n();
|
||||
|
||||
const props = defineProps({
|
||||
// 必填项
|
||||
content: {
|
||||
type: String,
|
||||
required: false
|
||||
|
|
@ -54,22 +77,88 @@ const props = defineProps({
|
|||
required: false
|
||||
}
|
||||
});
|
||||
|
||||
const showReferBox = ref(false);
|
||||
const expandedGroups = reactive({});
|
||||
|
||||
const referBoxSwitch = () => {
|
||||
showReferBox.value = !showReferBox.value;
|
||||
};
|
||||
|
||||
// 安全地处理内容
|
||||
const toggleGroup = (key) => {
|
||||
expandedGroups[key] = !expandedGroups[key];
|
||||
};
|
||||
|
||||
const webSearchRefs = computed(() => {
|
||||
if (!props.session?.knowledge_references) return [];
|
||||
return props.session.knowledge_references.filter(item => item.chunk_type === 'web_search');
|
||||
});
|
||||
|
||||
const knowledgeRefs = computed(() => {
|
||||
if (!props.session?.knowledge_references) return [];
|
||||
return props.session.knowledge_references.filter(item => item.chunk_type !== 'web_search');
|
||||
});
|
||||
|
||||
const groupedKnowledgeRefs = computed(() => {
|
||||
const refs = knowledgeRefs.value;
|
||||
if (!refs.length) return [];
|
||||
|
||||
const groupMap = new Map();
|
||||
for (const item of refs) {
|
||||
const key = item.knowledge_id || item.knowledge_title || item.id;
|
||||
if (!groupMap.has(key)) {
|
||||
groupMap.set(key, {
|
||||
key,
|
||||
title: item.knowledge_title || item.knowledge_filename || key,
|
||||
knowledgeId: item.knowledge_id,
|
||||
knowledgeBaseId: item.knowledge_base_id,
|
||||
chunks: [],
|
||||
});
|
||||
}
|
||||
groupMap.get(key).chunks.push(item);
|
||||
}
|
||||
return Array.from(groupMap.values());
|
||||
});
|
||||
|
||||
const headerText = computed(() => {
|
||||
const total = props.session?.knowledge_references?.length ?? 0;
|
||||
const docCount = groupedKnowledgeRefs.value.length;
|
||||
const webCount = webSearchRefs.value.length;
|
||||
if (docCount > 0 && webCount > 0) {
|
||||
return t('chat.referencesDocAndWebCount', { docCount, webCount });
|
||||
}
|
||||
if (docCount > 0) {
|
||||
return t('chat.referencesDocCount', { count: docCount });
|
||||
}
|
||||
return t('chat.referencesTitle', { count: total });
|
||||
});
|
||||
|
||||
const safeProcessContent = (content) => {
|
||||
if (!content) return '';
|
||||
// 先进行安全清理,然后处理换行
|
||||
const sanitized = sanitizeHTML(content);
|
||||
return sanitized.replace(/\n/g, '<br/>');
|
||||
};
|
||||
|
||||
// 获取 web_search 类型的 URL
|
||||
const truncateContent = (content, maxLen) => {
|
||||
if (!content) return '';
|
||||
const text = content.replace(/\n/g, ' ').trim();
|
||||
if (text.length <= maxLen) return text;
|
||||
return text.slice(0, maxLen) + '...';
|
||||
};
|
||||
|
||||
const navigateToDocument = (group) => {
|
||||
if (!group.knowledgeBaseId) return;
|
||||
const query = {};
|
||||
if (group.knowledgeId) {
|
||||
query.knowledge_id = group.knowledgeId;
|
||||
}
|
||||
router.push({
|
||||
path: `/platform/knowledge-bases/${group.knowledgeBaseId}`,
|
||||
query
|
||||
});
|
||||
};
|
||||
|
||||
const getWebSearchUrl = (item) => {
|
||||
// 优先使用 metadata.url,其次使用 id(如果 id 是 URL)
|
||||
if (item.metadata?.url) {
|
||||
return item.metadata.url;
|
||||
}
|
||||
|
|
@ -79,16 +168,13 @@ const getWebSearchUrl = (item) => {
|
|||
return '#';
|
||||
};
|
||||
|
||||
// 获取 web_search 类型的显示文本
|
||||
const getWebSearchDisplayText = (item) => {
|
||||
// 优先使用 knowledge_title,其次使用 metadata.title,最后使用 URL 的域名
|
||||
if (item.knowledge_title) {
|
||||
return item.knowledge_title;
|
||||
}
|
||||
if (item.metadata?.title) {
|
||||
return item.metadata.title;
|
||||
}
|
||||
// 如果都没有,使用 URL 的域名部分
|
||||
const url = getWebSearchUrl(item);
|
||||
if (url && url !== '#') {
|
||||
try {
|
||||
|
|
@ -100,7 +186,6 @@ const getWebSearchDisplayText = (item) => {
|
|||
}
|
||||
return 'Web Search Result';
|
||||
};
|
||||
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.refer {
|
||||
|
|
@ -155,23 +240,12 @@ const getWebSearchDisplayText = (item) => {
|
|||
}
|
||||
|
||||
.refer_box {
|
||||
padding: 4px 14px 4px 14px;
|
||||
padding: 4px 14px 8px 14px;
|
||||
flex-direction: column;
|
||||
border-top: 1px solid #f0f0f0;
|
||||
}
|
||||
}
|
||||
|
||||
.doc_content {
|
||||
max-height: 400px;
|
||||
overflow: auto;
|
||||
font-size: 14px;
|
||||
color: #000000e6;
|
||||
line-height: 23px;
|
||||
text-align: justify;
|
||||
border: 1px solid #07c05f33;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.doc {
|
||||
text-decoration: none;
|
||||
color: #07c05f;
|
||||
|
|
@ -185,28 +259,133 @@ const getWebSearchDisplayText = (item) => {
|
|||
padding: 2px 0;
|
||||
transition: all 0.2s ease;
|
||||
border-bottom: 1px solid transparent;
|
||||
|
||||
|
||||
&:hover {
|
||||
border-bottom-color: #07c05f;
|
||||
}
|
||||
|
||||
|
||||
&.doc-web {
|
||||
// Web search links can be longer, allow wrapping if needed
|
||||
white-space: normal;
|
||||
word-break: break-all;
|
||||
|
||||
|
||||
&:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.doc-group {
|
||||
margin-top: 4px;
|
||||
|
||||
.doc-group-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 4px 4px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
background-color: rgba(7, 192, 95, 0.04);
|
||||
}
|
||||
|
||||
.doc-group-left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.doc-group-arrow {
|
||||
color: #999;
|
||||
flex-shrink: 0;
|
||||
margin-right: 2px;
|
||||
}
|
||||
|
||||
.doc-group-icon {
|
||||
color: #07c05f;
|
||||
flex-shrink: 0;
|
||||
margin-right: 6px;
|
||||
}
|
||||
|
||||
.doc-group-title {
|
||||
color: #333;
|
||||
font-weight: 500;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
max-width: 200px;
|
||||
}
|
||||
|
||||
.doc-group-count {
|
||||
color: #999;
|
||||
font-size: 11px;
|
||||
margin-left: 6px;
|
||||
white-space: nowrap;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.doc-group-actions {
|
||||
flex-shrink: 0;
|
||||
margin-left: 8px;
|
||||
}
|
||||
|
||||
.doc-group-navigate {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
border-radius: 4px;
|
||||
color: #07c05f;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
background-color: rgba(7, 192, 95, 0.1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.doc-group-chunks {
|
||||
padding-left: 22px;
|
||||
}
|
||||
}
|
||||
|
||||
.doc-chunk-item {
|
||||
.doc-chunk-text {
|
||||
display: block;
|
||||
color: #666;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
padding: 3px 6px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.15s ease;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
|
||||
&:hover {
|
||||
background-color: rgba(7, 192, 95, 0.04);
|
||||
color: #07c05f;
|
||||
}
|
||||
|
||||
.doc-chunk-index {
|
||||
color: #999;
|
||||
font-size: 11px;
|
||||
margin-right: 4px;
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<style>
|
||||
.refer-to-layer {
|
||||
width: 400px;
|
||||
max-width: 500px;
|
||||
|
||||
|
||||
.t-popup__content {
|
||||
max-height: 400px;
|
||||
max-width: 500px;
|
||||
|
|
@ -216,4 +395,4 @@ const getWebSearchDisplayText = (item) => {
|
|||
word-break: break-word;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</style>
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import {
|
|||
import FAQEntryManager from './components/FAQEntryManager.vue';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { formatStringDate, kbFileTypeVerification } from '@/utils';
|
||||
import { getParserEngines, type ParserEngineInfo } from '@/api/system';
|
||||
const route = useRoute();
|
||||
const { t } = useI18n();
|
||||
const kbId = computed(() => (route.params as any).kbId as string || '');
|
||||
|
|
@ -40,6 +41,62 @@ const folderUploadInputRef = ref<HTMLInputElement | null>(null);
|
|||
const uploading = ref(false);
|
||||
const kbLoading = ref(false);
|
||||
const isFAQ = computed(() => (kbInfo.value?.type || '') === 'faq');
|
||||
const parserEngines = ref<ParserEngineInfo[]>([]);
|
||||
|
||||
const supportedFileTypes = computed<Set<string>>(() => {
|
||||
const engines = parserEngines.value
|
||||
if (!engines.length) return new Set<string>()
|
||||
|
||||
const rules: { file_types: string[]; engine: string }[] =
|
||||
kbInfo.value?.chunking_config?.parser_engine_rules || []
|
||||
|
||||
const ruleMap = new Map<string, string>()
|
||||
for (const r of rules) {
|
||||
for (const ft of r.file_types) ruleMap.set(ft, r.engine)
|
||||
}
|
||||
|
||||
const available = new Set<string>()
|
||||
const availableEngineNames = new Set(
|
||||
engines.filter(e => e.Available !== false).map(e => e.Name)
|
||||
)
|
||||
|
||||
for (const engine of engines) {
|
||||
for (const ft of engine.FileTypes || []) {
|
||||
if (available.has(ft)) continue
|
||||
|
||||
const explicitEngine = ruleMap.get(ft)
|
||||
if (explicitEngine) {
|
||||
if (availableEngineNames.has(explicitEngine)) available.add(ft)
|
||||
} else {
|
||||
if (engine.Available !== false) available.add(ft)
|
||||
}
|
||||
}
|
||||
}
|
||||
return available
|
||||
})
|
||||
|
||||
const acceptFileTypes = computed(() =>
|
||||
[...supportedFileTypes.value].map(t => '.' + t).join(',')
|
||||
)
|
||||
|
||||
const unsupportedFileTypes = computed<string[]>(() => {
|
||||
const engines = parserEngines.value
|
||||
if (!engines.length) return []
|
||||
|
||||
const allTypes = new Set<string>()
|
||||
for (const engine of engines) {
|
||||
for (const ft of engine.FileTypes || []) allTypes.add(ft)
|
||||
}
|
||||
|
||||
const supported = supportedFileTypes.value
|
||||
return [...allTypes].filter(ft => !supported.has(ft)).sort()
|
||||
})
|
||||
|
||||
const goToParserSettings = () => {
|
||||
if (kbId.value) {
|
||||
uiStore.openKBSettings(kbId.value, 'parser')
|
||||
}
|
||||
}
|
||||
|
||||
// Permission control: check if current user owns this KB or has edit/manage permission
|
||||
const isOwner = computed(() => {
|
||||
|
|
@ -121,6 +178,8 @@ const fileTypeOptions = computed(() => [
|
|||
{ content: 'PDF', value: 'pdf' },
|
||||
{ content: 'DOCX', value: 'docx' },
|
||||
{ content: 'DOC', value: 'doc' },
|
||||
{ content: 'PPTX', value: 'pptx' },
|
||||
{ content: 'PPT', value: 'ppt' },
|
||||
{ content: 'TXT', value: 'txt' },
|
||||
{ content: 'MD', value: 'md' },
|
||||
{ content: 'URL', value: 'url' },
|
||||
|
|
@ -564,15 +623,35 @@ const handleOpenURLImportDialog = (event: CustomEvent) => {
|
|||
}
|
||||
};
|
||||
|
||||
// Auto-open document detail when navigated with ?knowledge_id=xxx
|
||||
const pendingKnowledgeId = ref<string | null>(
|
||||
(route.query.knowledge_id as string) || null
|
||||
);
|
||||
|
||||
const tryAutoOpenDocument = () => {
|
||||
if (!pendingKnowledgeId.value || !cardList.value?.length) return;
|
||||
const targetId = pendingKnowledgeId.value;
|
||||
pendingKnowledgeId.value = null;
|
||||
const card = cardList.value.find((c: KnowledgeCard) => c.id === targetId);
|
||||
if (card) {
|
||||
nextTick(() => openCardDetails(card));
|
||||
} else {
|
||||
nextTick(() => {
|
||||
openCardDetails({ id: targetId } as KnowledgeCard);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
onMounted(() => {
|
||||
loadKnowledgeBaseInfo(kbId.value);
|
||||
loadKnowledgeList();
|
||||
// Load shared knowledge bases to get permission info
|
||||
orgStore.fetchSharedKnowledgeBases();
|
||||
|
||||
// 监听文件上传事件
|
||||
getParserEngines()
|
||||
.then(res => { parserEngines.value = res?.data || [] })
|
||||
.catch(() => { parserEngines.value = [] })
|
||||
|
||||
window.addEventListener('knowledgeFileUploaded', handleFileUploaded as EventListener);
|
||||
// 监听URL导入对话框打开事件
|
||||
window.addEventListener('openURLImportDialog', handleOpenURLImportDialog as EventListener);
|
||||
});
|
||||
|
||||
|
|
@ -582,6 +661,12 @@ onUnmounted(() => {
|
|||
});
|
||||
watch(() => cardList.value, (newValue) => {
|
||||
if (isFAQ.value) return;
|
||||
|
||||
// Auto-open document if navigated with ?knowledge_id=xxx
|
||||
if (pendingKnowledgeId.value && newValue?.length) {
|
||||
tryAutoOpenDocument();
|
||||
}
|
||||
|
||||
let analyzeList = [];
|
||||
// Filter items that need polling: parsing in progress OR summary generation in progress
|
||||
analyzeList = newValue.filter(item => {
|
||||
|
|
@ -780,21 +865,29 @@ const handleDocumentUpload = async (event: Event) => {
|
|||
return;
|
||||
}
|
||||
|
||||
// 过滤有效文件
|
||||
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
|
||||
const validFiles: File[] = [];
|
||||
let skippedCount = 0;
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
if (!kbFileTypeVerification(file, files.length > 1)) {
|
||||
if (!kbFileTypeVerification(file, files.length > 1, dynamicTypes)) {
|
||||
validFiles.push(file);
|
||||
} else {
|
||||
skippedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
if (skippedCount > 0) {
|
||||
MessagePlugin.warning(`所选文件类型暂无可用解析引擎,已全部跳过`);
|
||||
}
|
||||
resetUploadInput();
|
||||
return;
|
||||
}
|
||||
if (skippedCount > 0) {
|
||||
MessagePlugin.warning(`${skippedCount} 个文件因无可用解析引擎被跳过`);
|
||||
}
|
||||
|
||||
// 批量上传
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
const totalCount = validFiles.length;
|
||||
|
|
@ -871,10 +964,9 @@ const handleFolderUpload = async (event: Event) => {
|
|||
return;
|
||||
}
|
||||
|
||||
// 检查是否启用了VLM
|
||||
const vlmEnabled = kbInfo.value?.vlm_config?.enabled || false;
|
||||
const dynamicTypes = supportedFileTypes.value.size > 0 ? supportedFileTypes.value : undefined
|
||||
|
||||
// 过滤有效文件
|
||||
const validFiles: File[] = [];
|
||||
let hiddenFileCount = 0;
|
||||
let imageFilteredCount = 0;
|
||||
|
|
@ -883,7 +975,6 @@ const handleFolderUpload = async (event: Event) => {
|
|||
const file = files[i];
|
||||
const relativePath = (file as any).webkitRelativePath || file.name;
|
||||
|
||||
// 1. 过滤隐藏文件和隐藏文件夹
|
||||
const pathParts = relativePath.split('/');
|
||||
const hasHiddenComponent = pathParts.some((part: string) => part.startsWith('.'));
|
||||
if (hasHiddenComponent) {
|
||||
|
|
@ -891,7 +982,6 @@ const handleFolderUpload = async (event: Event) => {
|
|||
continue;
|
||||
}
|
||||
|
||||
// 2. 如果未启用VLM,过滤图片文件
|
||||
if (!vlmEnabled) {
|
||||
const fileExt = file.name.substring(file.name.lastIndexOf('.') + 1).toLowerCase();
|
||||
const imageTypes = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp'];
|
||||
|
|
@ -901,8 +991,7 @@ const handleFolderUpload = async (event: Event) => {
|
|||
}
|
||||
}
|
||||
|
||||
// 3. 文件类型验证
|
||||
if (!kbFileTypeVerification(file, true)) {
|
||||
if (!kbFileTypeVerification(file, true, dynamicTypes)) {
|
||||
validFiles.push(file);
|
||||
}
|
||||
}
|
||||
|
|
@ -1111,9 +1200,8 @@ const delCardConfirm = () => {
|
|||
|
||||
// 处理知识库编辑成功后的回调
|
||||
const handleKBEditorSuccess = (kbIdValue: string) => {
|
||||
// 如果编辑的是当前知识库,刷新文件列表
|
||||
if (kbIdValue === kbId.value) {
|
||||
loadKnowledgeFiles(kbIdValue);
|
||||
loadKnowledgeBaseInfo(kbIdValue);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -1226,6 +1314,11 @@ async function createNewSession(value: string): Promise<void> {
|
|||
</t-tooltip>
|
||||
</div>
|
||||
<p class="document-subtitle">{{ $t('knowledgeEditor.document.subtitle') }}</p>
|
||||
<p v-if="unsupportedFileTypes.length" class="parser-hint" @click="goToParserSettings">
|
||||
<t-icon name="info-circle" class="parser-hint-icon" />
|
||||
<span>{{ $t('knowledgeBase.unsupportedTypesHint', { types: unsupportedFileTypes.map(t => '.' + t).join('、') }) }}</span>
|
||||
<span class="parser-hint-link">{{ $t('knowledgeBase.goToParserSettings') }} →</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -1233,7 +1326,7 @@ async function createNewSession(value: string): Promise<void> {
|
|||
ref="uploadInputRef"
|
||||
type="file"
|
||||
class="document-upload-input"
|
||||
accept=".pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xlsx,.xls"
|
||||
:accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'"
|
||||
multiple
|
||||
@change="handleDocumentUpload"
|
||||
/>
|
||||
|
|
@ -2197,7 +2290,7 @@ async function createNewSession(value: string): Promise<void> {
|
|||
background: transparent;
|
||||
border: none;
|
||||
&:hover {
|
||||
color: #4e5969;
|
||||
color: #07C05F;
|
||||
background: #f2f3f5;
|
||||
}
|
||||
}
|
||||
|
|
@ -2212,7 +2305,7 @@ async function createNewSession(value: string): Promise<void> {
|
|||
&:hover,
|
||||
&:focus,
|
||||
&.t-is-focused {
|
||||
border-color: #4080ff;
|
||||
border-color: #07C05F;
|
||||
background-color: #fff;
|
||||
}
|
||||
}
|
||||
|
|
@ -2226,7 +2319,7 @@ async function createNewSession(value: string): Promise<void> {
|
|||
|
||||
&:hover,
|
||||
&.t-is-focused {
|
||||
border-color: #4080ff;
|
||||
border-color: #07C05F;
|
||||
background-color: #fff;
|
||||
}
|
||||
}
|
||||
|
|
@ -2375,6 +2468,36 @@ async function createNewSession(value: string): Promise<void> {
|
|||
line-height: 20px;
|
||||
}
|
||||
|
||||
.parser-hint {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
margin: 2px 0 0;
|
||||
color: #b08000;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
cursor: pointer;
|
||||
transition: color 0.15s ease;
|
||||
|
||||
&:hover {
|
||||
color: #8a6300;
|
||||
|
||||
.parser-hint-link {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
|
||||
.parser-hint-icon {
|
||||
font-size: 12px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.parser-hint-link {
|
||||
color: #07C05F;
|
||||
margin-left: 2px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -119,6 +119,22 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 解析引擎 -->
|
||||
<div v-if="!isFAQ && formData" v-show="currentSection === 'parser'" class="section">
|
||||
<KBParserSettings
|
||||
:parser-engine-rules="formData.chunkingConfig.parserEngineRules"
|
||||
@update:parser-engine-rules="handleParserEngineRulesUpdate"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- 存储引擎 -->
|
||||
<div v-if="!isFAQ && formData" v-show="currentSection === 'storage'" class="section">
|
||||
<KBStorageSettings
|
||||
:storage-provider="formData.storageProvider"
|
||||
@update:storage-provider="handleStorageProviderUpdate"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- 分块设置 -->
|
||||
<div v-if="!isFAQ" v-show="currentSection === 'chunking'" class="section">
|
||||
<KBChunkingSettings
|
||||
|
|
@ -182,6 +198,8 @@ import { updateKBConfig, type KBModelConfigRequest } from '@/api/initialization'
|
|||
import { listModels } from '@/api/model'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
import KBModelConfig from './settings/KBModelConfig.vue'
|
||||
import KBParserSettings from './settings/KBParserSettings.vue'
|
||||
import KBStorageSettings from './settings/KBStorageSettings.vue'
|
||||
import KBChunkingSettings from './settings/KBChunkingSettings.vue'
|
||||
import KBAdvancedSettings from './settings/KBAdvancedSettings.vue'
|
||||
import GraphSettings from './settings/GraphSettings.vue'
|
||||
|
|
@ -220,6 +238,8 @@ const navItems = computed(() => {
|
|||
items.push({ key: 'faq', icon: 'help-circle', label: t('knowledgeEditor.sidebar.faq') })
|
||||
} else {
|
||||
items.push(
|
||||
{ key: 'parser', icon: 'file-search', label: '解析引擎' },
|
||||
{ key: 'storage', icon: 'cloud', label: t('knowledgeEditor.sidebar.storage') },
|
||||
{ key: 'chunking', icon: 'file-copy', label: t('knowledgeEditor.sidebar.chunking') },
|
||||
{ key: 'graph', icon: 'chart-bubble', label: t('knowledgeEditor.sidebar.graph') },
|
||||
{ key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') }
|
||||
|
|
@ -274,25 +294,13 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
|
|||
chunkingConfig: {
|
||||
chunkSize: 512,
|
||||
chunkOverlap: 100,
|
||||
separators: ['\n\n', '\n', '。', '!', '?', ';', ';']
|
||||
separators: ['\n\n', '\n', '。', '!', '?', ';', ';'],
|
||||
parserEngineRules: undefined as any
|
||||
},
|
||||
storageProvider: 'local' as string,
|
||||
multimodalConfig: {
|
||||
enabled: false,
|
||||
storageType: 'minio' as 'minio' | 'cos',
|
||||
vllmModelId: '',
|
||||
minio: {
|
||||
bucketName: '',
|
||||
useSSL: false,
|
||||
pathPrefix: ''
|
||||
},
|
||||
cos: {
|
||||
secretId: '',
|
||||
secretKey: '',
|
||||
region: '',
|
||||
bucketName: '',
|
||||
appId: '',
|
||||
pathPrefix: ''
|
||||
}
|
||||
vllmModelId: ''
|
||||
},
|
||||
nodeExtractConfig: {
|
||||
enabled: false,
|
||||
|
|
@ -363,25 +371,13 @@ const loadKBData = async () => {
|
|||
chunkingConfig: {
|
||||
chunkSize: kb.chunking_config?.chunk_size || 512,
|
||||
chunkOverlap: kb.chunking_config?.chunk_overlap || 100,
|
||||
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';']
|
||||
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'],
|
||||
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined
|
||||
},
|
||||
storageProvider: (kb.cos_config?.provider || 'local') as string,
|
||||
multimodalConfig: {
|
||||
enabled: !!(kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)),
|
||||
storageType: (kb.cos_config?.provider || 'minio') as 'minio' | 'cos',
|
||||
vllmModelId: kb.vlm_config?.model_id || '',
|
||||
minio: {
|
||||
bucketName: kb.cos_config?.bucket_name || '',
|
||||
useSSL: kb.cos_config?.use_ssl || false,
|
||||
pathPrefix: kb.cos_config?.path_prefix || ''
|
||||
},
|
||||
cos: {
|
||||
secretId: kb.cos_config?.secret_id || '',
|
||||
secretKey: kb.cos_config?.secret_key || '',
|
||||
region: kb.cos_config?.region || '',
|
||||
bucketName: kb.cos_config?.bucket_name || '',
|
||||
appId: kb.cos_config?.app_id || '',
|
||||
pathPrefix: kb.cos_config?.path_prefix || ''
|
||||
}
|
||||
enabled: !!kb.vlm_config?.enabled,
|
||||
vllmModelId: kb.vlm_config?.model_id || ''
|
||||
},
|
||||
nodeExtractConfig: {
|
||||
enabled: kb.extract_config?.enabled || false,
|
||||
|
|
@ -420,12 +416,24 @@ const handleChunkingConfigUpdate = (config: any) => {
|
|||
}
|
||||
}
|
||||
|
||||
const handleParserEngineRulesUpdate = (rules: any[]) => {
|
||||
if (formData.value) {
|
||||
formData.value.chunkingConfig.parserEngineRules = rules?.length ? rules : undefined
|
||||
}
|
||||
}
|
||||
|
||||
const handleMultimodalUpdate = (config: any) => {
|
||||
if (formData.value) {
|
||||
formData.value.multimodalConfig = { ...config }
|
||||
}
|
||||
}
|
||||
|
||||
const handleStorageProviderUpdate = (value: string) => {
|
||||
if (formData.value) {
|
||||
formData.value.storageProvider = value || 'local'
|
||||
}
|
||||
}
|
||||
|
||||
const handleQuestionGenerationUpdate = (config: any) => {
|
||||
if (formData.value) {
|
||||
formData.value.questionGenerationConfig = { ...config }
|
||||
|
|
@ -493,7 +501,10 @@ const buildSubmitData = () => {
|
|||
chunk_size: formData.value.chunkingConfig.chunkSize,
|
||||
chunk_overlap: formData.value.chunkingConfig.chunkOverlap,
|
||||
separators: formData.value.chunkingConfig.separators,
|
||||
enable_multimodal: formData.value.multimodalConfig.enabled
|
||||
enable_multimodal: formData.value.multimodalConfig.enabled,
|
||||
...(formData.value.chunkingConfig.parserEngineRules?.length
|
||||
? { parser_engine_rules: formData.value.chunkingConfig.parserEngineRules }
|
||||
: {})
|
||||
},
|
||||
embedding_model_id: formData.value.modelConfig.embeddingModelId,
|
||||
summary_model_id: formData.value.modelConfig.llmModelId
|
||||
|
|
@ -507,26 +518,9 @@ const buildSubmitData = () => {
|
|||
: ''
|
||||
}
|
||||
|
||||
if (formData.value.multimodalConfig.enabled) {
|
||||
const storageType = formData.value.multimodalConfig.storageType
|
||||
if (storageType === 'minio') {
|
||||
data.cos_config = {
|
||||
provider: 'minio',
|
||||
bucket_name: formData.value.multimodalConfig.minio.bucketName,
|
||||
use_ssl: formData.value.multimodalConfig.minio.useSSL,
|
||||
path_prefix: formData.value.multimodalConfig.minio.pathPrefix || undefined
|
||||
}
|
||||
} else {
|
||||
data.cos_config = {
|
||||
provider: 'cos',
|
||||
secret_id: formData.value.multimodalConfig.cos.secretId,
|
||||
secret_key: formData.value.multimodalConfig.cos.secretKey,
|
||||
region: formData.value.multimodalConfig.cos.region,
|
||||
bucket_name: formData.value.multimodalConfig.cos.bucketName,
|
||||
app_id: formData.value.multimodalConfig.cos.appId,
|
||||
path_prefix: formData.value.multimodalConfig.cos.pathPrefix || undefined
|
||||
}
|
||||
}
|
||||
// 存储引擎:仅传 provider,参数从全局设置读取
|
||||
data.cos_config = {
|
||||
provider: formData.value.storageProvider || 'local'
|
||||
}
|
||||
|
||||
// 添加知识图谱配置
|
||||
|
|
@ -599,7 +593,7 @@ const handleSubmit = async () => {
|
|||
config: updateConfig
|
||||
})
|
||||
|
||||
// 2. 更新完整配置(模型、分块、多模态、知识图谱等)
|
||||
// 2. 更新完整配置(模型、分块、多模态、存储引擎、知识图谱等)
|
||||
const config: KBModelConfigRequest = {
|
||||
llmModelId: data.summary_model_id,
|
||||
embeddingModelId: data.embedding_model_id,
|
||||
|
|
@ -607,25 +601,13 @@ const handleSubmit = async () => {
|
|||
documentSplitting: {
|
||||
chunkSize: data.chunking_config.chunk_size,
|
||||
chunkOverlap: data.chunking_config.chunk_overlap,
|
||||
separators: data.chunking_config.separators
|
||||
separators: data.chunking_config.separators,
|
||||
parserEngineRules: data.chunking_config.parser_engine_rules || undefined
|
||||
},
|
||||
multimodal: {
|
||||
enabled: !!data.cos_config || !!data.vlm_config?.enabled,
|
||||
storageType: data.cos_config?.provider || 'minio',
|
||||
cos: data.cos_config?.provider === 'cos' ? {
|
||||
secretId: data.cos_config.secret_id,
|
||||
secretKey: data.cos_config.secret_key,
|
||||
region: data.cos_config.region,
|
||||
bucketName: data.cos_config.bucket_name,
|
||||
appId: data.cos_config.app_id,
|
||||
pathPrefix: data.cos_config.path_prefix || ''
|
||||
} : undefined,
|
||||
minio: data.cos_config?.provider === 'minio' ? {
|
||||
bucketName: data.cos_config.bucket_name,
|
||||
useSSL: data.cos_config.use_ssl || false,
|
||||
pathPrefix: data.cos_config.path_prefix || ''
|
||||
} : undefined
|
||||
enabled: !!data.vlm_config?.enabled
|
||||
},
|
||||
storageProvider: data.cos_config?.provider || 'local',
|
||||
nodeExtract: {
|
||||
enabled: data.extract_config?.enabled || false,
|
||||
text: data.extract_config?.text || '',
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Multimodal feature -->
|
||||
<!-- Multimodal feature:仅选择多模态模型,存储引擎在「存储引擎」页配置 -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.label') }}</label>
|
||||
|
|
@ -57,9 +57,8 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Multimodal storage configuration -->
|
||||
<!-- 多模态开启时仅配置 VLLM 模型 -->
|
||||
<div v-if="localMultimodal.enabled" class="subsection">
|
||||
<!-- VLLM model -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.vllmLabel') }} <span class="required">*</span></label>
|
||||
|
|
@ -77,206 +76,6 @@
|
|||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="subsection-header">
|
||||
<h4>{{ $t('knowledgeEditor.advanced.multimodal.storageTitle') }} <span class="required">*</span></h4>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.storageTypeLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.storageTypeDescription') }}</p>
|
||||
<!-- Warning message when MinIO is not enabled -->
|
||||
<t-alert
|
||||
v-if="!isMinioEnabled"
|
||||
theme="warning"
|
||||
:message="$t('knowledgeEditor.advanced.multimodal.minioDisabledWarning')"
|
||||
style="margin-top: 8px;"
|
||||
/>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-radio-group v-model="localMultimodal.storageType" @change="handleStorageTypeChange">
|
||||
<t-radio value="minio" :disabled="!isMinioEnabled">
|
||||
{{ $t('knowledgeEditor.advanced.multimodal.storageTypeOptions.minio') }}
|
||||
</t-radio>
|
||||
<t-radio value="cos">{{ $t('knowledgeEditor.advanced.multimodal.storageTypeOptions.cos') }}</t-radio>
|
||||
</t-radio-group>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- MinIO configuration -->
|
||||
<div v-if="localMultimodal.storageType === 'minio'" class="storage-config">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.minio.bucketLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.minio.bucketDescription') }}</p>
|
||||
<p class="hint">{{ $t('knowledgeEditor.advanced.multimodal.minio.bucketHint') }}</p>
|
||||
</div>
|
||||
<div class="setting-control bucket-control">
|
||||
<t-select
|
||||
v-model="localMultimodal.minio.bucketName"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.minio.bucketPlaceholder')"
|
||||
:loading="loadingBuckets"
|
||||
filterable
|
||||
creatable
|
||||
@change="handleConfigChange"
|
||||
@focus="loadMinioBuckets"
|
||||
style="width: 280px;"
|
||||
>
|
||||
<t-option
|
||||
v-for="bucket in minioBuckets"
|
||||
:key="bucket.name"
|
||||
:value="bucket.name"
|
||||
:label="bucket.name"
|
||||
>
|
||||
<div class="bucket-option">
|
||||
<span class="bucket-name">{{ bucket.name }}</span>
|
||||
<t-tag
|
||||
:theme="bucket.policy === 'public' ? 'success' : bucket.policy === 'private' ? 'default' : 'warning'"
|
||||
size="small"
|
||||
variant="light"
|
||||
>
|
||||
{{ $t(`knowledgeEditor.advanced.multimodal.minio.policyLabels.${bucket.policy}`) }}
|
||||
</t-tag>
|
||||
</div>
|
||||
</t-option>
|
||||
</t-select>
|
||||
<t-button
|
||||
theme="default"
|
||||
variant="outline"
|
||||
size="small"
|
||||
:loading="loadingBuckets"
|
||||
@click="loadMinioBuckets"
|
||||
style="margin-left: 8px;"
|
||||
>
|
||||
<template #icon><refresh-icon /></template>
|
||||
</t-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.minio.useSslLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.minio.useSslDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-switch
|
||||
v-model="localMultimodal.minio.useSSL"
|
||||
@change="handleConfigChange"
|
||||
size="large"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.minio.pathPrefixLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.minio.pathPrefixDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.minio.pathPrefix"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.minio.pathPrefixPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- COS configuration -->
|
||||
<div v-if="localMultimodal.storageType === 'cos'" class="storage-config">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.secretIdLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.secretIdDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.cos.secretId"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.secretIdPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.secretKeyLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.secretKeyDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.cos.secretKey"
|
||||
type="password"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.secretKeyPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.regionLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.regionDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.cos.region"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.regionPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.bucketLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.bucketDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.cos.bucketName"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.bucketPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.appIdLabel') }} <span class="required">*</span></label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.appIdDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.cos.appId"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.appIdPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.multimodal.cos.pathPrefixLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.multimodal.cos.pathPrefixDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input
|
||||
v-model="localMultimodal.cos.pathPrefix"
|
||||
:placeholder="$t('knowledgeEditor.advanced.multimodal.cos.pathPrefixPlaceholder')"
|
||||
@change="handleConfigChange"
|
||||
style="width: 280px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
@ -284,31 +83,15 @@
|
|||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, onMounted } from 'vue'
|
||||
import { RefreshIcon } from 'tdesign-icons-vue-next'
|
||||
import { ref, watch } from 'vue'
|
||||
import ModelSelector from '@/components/ModelSelector.vue'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
import { getSystemInfo, listMinioBuckets, type MinioBucketInfo } from '@/api/system'
|
||||
|
||||
const uiStore = useUIStore()
|
||||
|
||||
interface MultimodalConfig {
|
||||
enabled: boolean
|
||||
storageType: 'minio' | 'cos'
|
||||
vllmModelId?: string
|
||||
minio: {
|
||||
bucketName: string
|
||||
useSSL: boolean
|
||||
pathPrefix: string
|
||||
}
|
||||
cos: {
|
||||
secretId: string
|
||||
secretKey: string
|
||||
region: string
|
||||
bucketName: string
|
||||
appId: string
|
||||
pathPrefix: string
|
||||
}
|
||||
}
|
||||
|
||||
interface QuestionGenerationConfig {
|
||||
|
|
@ -335,58 +118,7 @@ const localQuestionGeneration = ref<QuestionGenerationConfig>(
|
|||
)
|
||||
|
||||
const vllmSelectorRef = ref()
|
||||
const isMinioEnabled = ref(false)
|
||||
const minioBuckets = ref<MinioBucketInfo[]>([])
|
||||
const loadingBuckets = ref(false)
|
||||
|
||||
// Load MinIO buckets
|
||||
const loadMinioBuckets = async () => {
|
||||
if (!isMinioEnabled.value || loadingBuckets.value) return
|
||||
|
||||
loadingBuckets.value = true
|
||||
try {
|
||||
const response = await listMinioBuckets()
|
||||
if (response.data?.buckets) {
|
||||
minioBuckets.value = response.data.buckets
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load MinIO buckets:', error)
|
||||
} finally {
|
||||
loadingBuckets.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// Check system status on mount
|
||||
onMounted(async () => {
|
||||
try {
|
||||
const systemInfo = await getSystemInfo()
|
||||
|
||||
// Check MinIO status
|
||||
isMinioEnabled.value = systemInfo.data?.minio_enabled === true
|
||||
|
||||
// If MinIO is not enabled and storage type is minio, switch to cos
|
||||
if (!isMinioEnabled.value && localMultimodal.value.storageType === 'minio') {
|
||||
localMultimodal.value.storageType = 'cos'
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
// Load MinIO buckets if enabled
|
||||
if (isMinioEnabled.value) {
|
||||
loadMinioBuckets()
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch system info:', error)
|
||||
// Default to disabled if we can't fetch the info
|
||||
isMinioEnabled.value = false
|
||||
// If MinIO status unknown and storage type is minio, switch to cos
|
||||
if (localMultimodal.value.storageType === 'minio') {
|
||||
localMultimodal.value.storageType = 'cos'
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Watch for prop changes
|
||||
watch(() => props.multimodal, (newVal) => {
|
||||
localMultimodal.value = { ...newVal }
|
||||
}, { deep: true })
|
||||
|
|
@ -397,7 +129,6 @@ watch(() => props.questionGeneration, (newVal) => {
|
|||
}
|
||||
}, { deep: true })
|
||||
|
||||
// Handle question generation toggle
|
||||
const handleQuestionGenerationToggle = () => {
|
||||
if (!localQuestionGeneration.value.enabled) {
|
||||
localQuestionGeneration.value.questionCount = 3
|
||||
|
|
@ -405,59 +136,25 @@ const handleQuestionGenerationToggle = () => {
|
|||
emit('update:questionGeneration', localQuestionGeneration.value)
|
||||
}
|
||||
|
||||
// Handle question generation config change
|
||||
const handleQuestionGenerationChange = () => {
|
||||
emit('update:questionGeneration', localQuestionGeneration.value)
|
||||
}
|
||||
|
||||
// Handle multimodal toggle
|
||||
const handleMultimodalToggle = () => {
|
||||
// Reset related configuration when multimodal is disabled
|
||||
if (!localMultimodal.value.enabled) {
|
||||
localMultimodal.value.vllmModelId = ''
|
||||
localMultimodal.value.minio = {
|
||||
bucketName: '',
|
||||
useSSL: false,
|
||||
pathPrefix: ''
|
||||
}
|
||||
localMultimodal.value.cos = {
|
||||
secretId: '',
|
||||
secretKey: '',
|
||||
region: '',
|
||||
bucketName: '',
|
||||
appId: '',
|
||||
pathPrefix: ''
|
||||
}
|
||||
}
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
// Handle storage type change
|
||||
const handleStorageTypeChange = () => {
|
||||
// Prevent switching to minio if it's not enabled
|
||||
if (localMultimodal.value.storageType === 'minio' && !isMinioEnabled.value) {
|
||||
localMultimodal.value.storageType = 'cos'
|
||||
}
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
// Handle VLLM model change
|
||||
const handleVLLMChange = (modelId: string) => {
|
||||
localMultimodal.value.vllmModelId = modelId
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
// Navigate to model management when adding models
|
||||
const handleAddModel = (subSection: string) => {
|
||||
uiStore.openSettings('models', subSection)
|
||||
}
|
||||
|
||||
// Handle configuration change
|
||||
const handleConfigChange = () => {
|
||||
emit('update:multimodal', localMultimodal.value)
|
||||
}
|
||||
|
||||
// The allModels prop keeps model options in sync
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
|
|
@ -546,104 +243,11 @@ const handleConfigChange = () => {
|
|||
position: relative;
|
||||
}
|
||||
|
||||
.subsection-header {
|
||||
margin: 16px 0 8px 0;
|
||||
|
||||
&:first-child {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
h4 {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: #333333;
|
||||
margin: 0;
|
||||
padding-left: 8px;
|
||||
border-left: 2px solid #07C05F;
|
||||
|
||||
.required {
|
||||
color: #e34d59;
|
||||
margin-left: 4px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.required {
|
||||
color: #e34d59;
|
||||
margin-left: 2px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.storage-config {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.bucket-control {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.bucket-option {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
width: 100%;
|
||||
|
||||
.bucket-name {
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.config-item {
|
||||
background: #ffffff;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 6px;
|
||||
padding: 16px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.config-item-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 16px;
|
||||
padding-bottom: 12px;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
}
|
||||
|
||||
.config-item-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
.config-item-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.config-field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
|
||||
label {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: #555555;
|
||||
}
|
||||
}
|
||||
|
||||
.subsection-desc {
|
||||
font-size: 13px;
|
||||
color: #666666;
|
||||
margin: 4px 0 8px 8px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
|
|
|
|||
|
|
@ -77,10 +77,16 @@
|
|||
import { ref, watch, computed } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
interface ParserEngineRule {
|
||||
file_types: string[]
|
||||
engine: string
|
||||
}
|
||||
|
||||
interface ChunkingConfig {
|
||||
chunkSize: number
|
||||
chunkOverlap: number
|
||||
separators: string[]
|
||||
parserEngineRules?: ParserEngineRule[]
|
||||
}
|
||||
|
||||
interface Props {
|
||||
|
|
@ -98,7 +104,6 @@ const localChunkOverlap = ref(props.config.chunkOverlap)
|
|||
const localSeparators = ref([...props.config.separators])
|
||||
const { t } = useI18n()
|
||||
|
||||
// Separator options
|
||||
const separatorOptions = computed(() => [
|
||||
{ label: t('knowledgeEditor.chunking.separators.doubleNewline'), value: '\n\n' },
|
||||
{ label: t('knowledgeEditor.chunking.separators.singleNewline'), value: '\n' },
|
||||
|
|
@ -110,34 +115,22 @@ const separatorOptions = computed(() => [
|
|||
{ label: t('knowledgeEditor.chunking.separators.space'), value: ' ' }
|
||||
])
|
||||
|
||||
// Watch for prop changes
|
||||
watch(() => props.config, (newConfig) => {
|
||||
localChunkSize.value = newConfig.chunkSize
|
||||
localChunkOverlap.value = newConfig.chunkOverlap
|
||||
localSeparators.value = [...newConfig.separators]
|
||||
}, { deep: true })
|
||||
|
||||
// Handle chunk size change
|
||||
const handleChunkSizeChange = () => {
|
||||
emitUpdate()
|
||||
}
|
||||
const handleChunkSizeChange = () => { emitUpdate() }
|
||||
const handleChunkOverlapChange = () => { emitUpdate() }
|
||||
const handleSeparatorsChange = () => { emitUpdate() }
|
||||
|
||||
// Handle chunk overlap change
|
||||
const handleChunkOverlapChange = () => {
|
||||
emitUpdate()
|
||||
}
|
||||
|
||||
// Handle separator change
|
||||
const handleSeparatorsChange = () => {
|
||||
emitUpdate()
|
||||
}
|
||||
|
||||
// Emit update event
|
||||
const emitUpdate = () => {
|
||||
emit('update:config', {
|
||||
chunkSize: localChunkSize.value,
|
||||
chunkOverlap: localChunkOverlap.value,
|
||||
separators: localSeparators.value
|
||||
separators: localSeparators.value,
|
||||
parserEngineRules: props.config.parserEngineRules
|
||||
})
|
||||
}
|
||||
</script>
|
||||
|
|
@ -228,4 +221,3 @@ const emitUpdate = () => {
|
|||
text-align: right;
|
||||
}
|
||||
</style>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,456 @@
|
|||
<template>
|
||||
<div class="kb-parser-settings">
|
||||
<div class="section-header">
|
||||
<h2>解析引擎</h2>
|
||||
<p class="section-description">为不同文件类型选择文档解析引擎。未配置的文件类型将使用内置解析引擎。</p>
|
||||
</div>
|
||||
|
||||
<div v-if="loading" class="loading-inline">
|
||||
<t-loading size="small" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
|
||||
<div v-else-if="fileTypeGroups.length === 0" class="empty-hint">
|
||||
<p>暂无可用解析引擎,或文档解析服务未配置。</p>
|
||||
</div>
|
||||
|
||||
<div v-else class="settings-group">
|
||||
<div
|
||||
v-for="group in fileTypeGroups"
|
||||
:key="group.key"
|
||||
class="setting-row"
|
||||
>
|
||||
<div class="setting-info">
|
||||
<label class="group-label">
|
||||
<t-icon :name="group.icon" class="group-icon" />
|
||||
{{ group.label }}
|
||||
</label>
|
||||
<div class="ext-tags">
|
||||
<span v-for="ext in group.extensions" :key="ext" class="ext-tag">.{{ ext }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-select
|
||||
:value="getEngineForGroup(group.extensions) || undefined"
|
||||
@change="(val: string) => handleEngineChange(group.extensions, val)"
|
||||
style="width: 280px;"
|
||||
:status="hasAvailableEngine(group.extensions) ? 'default' : 'warning'"
|
||||
placeholder="无可用引擎"
|
||||
>
|
||||
<t-option
|
||||
v-for="opt in getEngineOptions(group.extensions)"
|
||||
:key="opt.value"
|
||||
:value="opt.value"
|
||||
:label="opt.selectLabel"
|
||||
:disabled="opt.disabled"
|
||||
>
|
||||
<t-tooltip
|
||||
:content="`支持格式: ${opt.fileTypes.map(t => '.' + t).join(' ')}`"
|
||||
placement="left"
|
||||
:show-arrow="false"
|
||||
>
|
||||
<div class="engine-option">
|
||||
<div class="engine-option-top">
|
||||
<span class="engine-option-name">{{ opt.value }}</span>
|
||||
<t-tag
|
||||
v-if="opt.isDefault"
|
||||
theme="primary"
|
||||
variant="light"
|
||||
size="small"
|
||||
>默认</t-tag>
|
||||
<t-tag
|
||||
v-if="opt.disabled"
|
||||
theme="danger"
|
||||
variant="light"
|
||||
size="small"
|
||||
>不可用</t-tag>
|
||||
</div>
|
||||
<div class="engine-option-desc">{{ opt.desc }}</div>
|
||||
<div v-if="opt.disabled && opt.reason" class="engine-option-reason">
|
||||
{{ opt.reason }}
|
||||
<a class="go-settings" @click.stop.prevent="goToParserSettings">去设置 →</a>
|
||||
</div>
|
||||
</div>
|
||||
</t-tooltip>
|
||||
</t-option>
|
||||
</t-select>
|
||||
<div v-if="!hasAvailableEngine(group.extensions)" class="no-engine-warning">
|
||||
<a class="go-settings" @click.prevent="goToParserSettings">前往配置 →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, computed, onMounted, onUnmounted } from 'vue'
|
||||
import { getParserEngines, type ParserEngineInfo } from '@/api/system'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
|
||||
export interface ParserEngineRule {
|
||||
file_types: string[]
|
||||
engine: string
|
||||
}
|
||||
|
||||
interface EngineOption {
|
||||
value: string
|
||||
selectLabel: string
|
||||
desc: string
|
||||
fileTypes: string[]
|
||||
disabled: boolean
|
||||
isDefault: boolean
|
||||
reason?: string
|
||||
}
|
||||
|
||||
interface Props {
|
||||
parserEngineRules?: ParserEngineRule[]
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
parserEngineRules: () => []
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:parserEngineRules': [value: ParserEngineRule[]]
|
||||
}>()
|
||||
|
||||
const uiStore = useUIStore()
|
||||
const localEngineRules = ref<ParserEngineRule[]>([...props.parserEngineRules])
|
||||
const parserEngines = ref<ParserEngineInfo[]>([])
|
||||
const loading = ref(true)
|
||||
|
||||
const allFileTypes = computed(() => {
|
||||
const s = new Set<string>()
|
||||
for (const engine of parserEngines.value) {
|
||||
for (const ft of engine.FileTypes || []) {
|
||||
s.add(ft)
|
||||
}
|
||||
}
|
||||
return s
|
||||
})
|
||||
|
||||
const fileTypeGroups = computed(() => {
|
||||
const ft = allFileTypes.value
|
||||
const groups: { key: string; label: string; icon: string; extensions: string[] }[] = []
|
||||
|
||||
const pdfExts = ['pdf'].filter(e => ft.has(e))
|
||||
const officeExts = ['docx', 'doc'].filter(e => ft.has(e))
|
||||
const pptExts = ['pptx', 'ppt'].filter(e => ft.has(e))
|
||||
const excelExts = ['xlsx', 'xls'].filter(e => ft.has(e))
|
||||
const csvExts = ['csv'].filter(e => ft.has(e))
|
||||
const mdExts = ['md', 'markdown'].filter(e => ft.has(e))
|
||||
const txtExts = ['txt'].filter(e => ft.has(e))
|
||||
const imageExts = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp'].filter(e => ft.has(e))
|
||||
|
||||
if (pdfExts.length) groups.push({ key: 'pdf', label: 'PDF 文档', icon: 'file-pdf', extensions: pdfExts })
|
||||
if (officeExts.length) groups.push({ key: 'office', label: 'Word 文档', icon: 'file-word', extensions: officeExts })
|
||||
if (pptExts.length) groups.push({ key: 'ppt', label: '演示文稿', icon: 'file-powerpoint', extensions: pptExts })
|
||||
if (excelExts.length) groups.push({ key: 'excel', label: 'Excel 表格', icon: 'file-excel', extensions: excelExts })
|
||||
if (csvExts.length) groups.push({ key: 'csv', label: 'CSV 文件', icon: 'file-excel', extensions: csvExts })
|
||||
if (mdExts.length) groups.push({ key: 'markdown', label: 'Markdown', icon: 'file-code', extensions: mdExts })
|
||||
if (txtExts.length) groups.push({ key: 'text', label: '纯文本', icon: 'file', extensions: txtExts })
|
||||
if (imageExts.length) groups.push({ key: 'image', label: '图片', icon: 'image', extensions: imageExts })
|
||||
|
||||
return groups
|
||||
})
|
||||
|
||||
function getEngineOptions(extensions: string[]): EngineOption[] {
|
||||
const raw: { name: string; desc: string; fileTypes: string[]; available: boolean; reason: string }[] = []
|
||||
for (const engine of parserEngines.value) {
|
||||
const supports = extensions.some(ext => (engine.FileTypes || []).includes(ext))
|
||||
if (supports) {
|
||||
raw.push({
|
||||
name: engine.Name,
|
||||
desc: engine.Description || engine.Name,
|
||||
fileTypes: engine.FileTypes || [],
|
||||
available: engine.Available !== false,
|
||||
reason: engine.UnavailableReason || '',
|
||||
})
|
||||
}
|
||||
}
|
||||
const defaultName = raw.find(e => e.available)?.name ?? ''
|
||||
return raw.map(e => ({
|
||||
value: e.name,
|
||||
selectLabel: `${e.name} — ${e.desc}`,
|
||||
desc: e.desc,
|
||||
fileTypes: e.fileTypes,
|
||||
disabled: !e.available,
|
||||
isDefault: defaultName !== '' && e.name === defaultName,
|
||||
reason: e.reason,
|
||||
}))
|
||||
}
|
||||
|
||||
function hasAvailableEngine(extensions: string[]): boolean {
|
||||
return getEngineOptions(extensions).some(opt => !opt.disabled)
|
||||
}
|
||||
|
||||
function getDefaultEngine(extensions: string[]): string {
|
||||
const opts = getEngineOptions(extensions)
|
||||
return opts.find(o => o.isDefault)?.value ?? ''
|
||||
}
|
||||
|
||||
function getEngineForGroup(extensions: string[]): string {
|
||||
for (const rule of localEngineRules.value) {
|
||||
if (rule.file_types.some(ft => extensions.includes(ft))) {
|
||||
return rule.engine
|
||||
}
|
||||
}
|
||||
return getDefaultEngine(extensions)
|
||||
}
|
||||
|
||||
function handleEngineChange(extensions: string[], engine: string) {
|
||||
const otherRules = localEngineRules.value.filter(
|
||||
r => !r.file_types.some(ft => extensions.includes(ft))
|
||||
)
|
||||
if (engine) {
|
||||
otherRules.push({ file_types: [...extensions], engine })
|
||||
}
|
||||
localEngineRules.value = otherRules
|
||||
emit('update:parserEngineRules', buildCompleteRules())
|
||||
}
|
||||
|
||||
function buildCompleteRules(): ParserEngineRule[] {
|
||||
const rules: ParserEngineRule[] = []
|
||||
for (const group of fileTypeGroups.value) {
|
||||
const engine = getEngineForGroup(group.extensions)
|
||||
if (engine) {
|
||||
rules.push({ file_types: [...group.extensions], engine })
|
||||
}
|
||||
}
|
||||
return rules
|
||||
}
|
||||
|
||||
function goToParserSettings() {
|
||||
uiStore.openSettings('parser')
|
||||
}
|
||||
|
||||
async function loadEngines() {
|
||||
loading.value = true
|
||||
try {
|
||||
const resp = await getParserEngines()
|
||||
if (resp?.data && Array.isArray(resp.data)) {
|
||||
parserEngines.value = resp.data
|
||||
}
|
||||
} catch {
|
||||
parserEngines.value = []
|
||||
} finally {
|
||||
loading.value = false
|
||||
ensureCompleteRules()
|
||||
}
|
||||
}
|
||||
|
||||
function ensureCompleteRules() {
|
||||
if (!parserEngines.value.length) return
|
||||
const complete = buildCompleteRules()
|
||||
if (complete.length && complete.length > localEngineRules.value.length) {
|
||||
localEngineRules.value = complete
|
||||
emit('update:parserEngineRules', complete)
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(loadEngines)
|
||||
|
||||
const { showSettingsModal } = storeToRefs(uiStore)
|
||||
watch(showSettingsModal, (open, wasOpen) => {
|
||||
if (wasOpen && !open) {
|
||||
loadEngines()
|
||||
}
|
||||
})
|
||||
|
||||
watch(() => props.parserEngineRules, (v) => {
|
||||
localEngineRules.value = v?.length ? [...v] : []
|
||||
}, { deep: true })
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.kb-parser-settings {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
margin-bottom: 32px;
|
||||
|
||||
h2 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin: 0 0 8px 0;
|
||||
}
|
||||
|
||||
.section-description {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.loading-inline {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 16px 0;
|
||||
}
|
||||
|
||||
.empty-hint {
|
||||
padding: 24px 0;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.settings-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.setting-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
padding: 20px 0;
|
||||
border-bottom: 1px solid #e5e7eb;
|
||||
|
||||
&:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
}
|
||||
|
||||
.setting-info {
|
||||
flex: 1;
|
||||
max-width: 65%;
|
||||
padding-right: 24px;
|
||||
|
||||
.group-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.group-icon {
|
||||
font-size: 18px;
|
||||
color: #555;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
label {
|
||||
font-size: 15px;
|
||||
font-weight: 500;
|
||||
color: #333;
|
||||
display: block;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.ext-tags {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.ext-tag {
|
||||
display: inline-block;
|
||||
font-size: 12px;
|
||||
line-height: 1;
|
||||
color: #555;
|
||||
background: #f3f4f6;
|
||||
padding: 3px 8px;
|
||||
border-radius: 4px;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
|
||||
}
|
||||
|
||||
.desc {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex-shrink: 0;
|
||||
min-width: 280px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-end;
|
||||
}
|
||||
|
||||
.no-engine-warning {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
margin-top: 8px;
|
||||
font-size: 12px;
|
||||
color: #e37318;
|
||||
line-height: 1.4;
|
||||
|
||||
.go-settings {
|
||||
color: #07C05F;
|
||||
cursor: pointer;
|
||||
white-space: nowrap;
|
||||
text-decoration: none;
|
||||
|
||||
&:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 下拉选项样式 ----
|
||||
.engine-option {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
padding: 3px 0;
|
||||
}
|
||||
|
||||
.engine-option-top {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.engine-option-name {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
|
||||
}
|
||||
|
||||
.engine-option-desc {
|
||||
font-size: 12px;
|
||||
color: #888;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.engine-option-reason {
|
||||
font-size: 12px;
|
||||
color: #e34d59;
|
||||
line-height: 1.4;
|
||||
|
||||
.go-settings {
|
||||
color: #07C05F;
|
||||
cursor: pointer;
|
||||
margin-left: 4px;
|
||||
font-size: 12px;
|
||||
text-decoration: none;
|
||||
|
||||
&:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
<style lang="less">
|
||||
.t-select__dropdown .t-select-option {
|
||||
height: auto;
|
||||
align-items: flex-start;
|
||||
padding-top: 6px;
|
||||
padding-bottom: 6px;
|
||||
}
|
||||
.t-select__dropdown .t-select-option__content {
|
||||
white-space: normal;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,244 @@
|
|||
<template>
|
||||
<div class="kb-storage-settings">
|
||||
<div class="section-header">
|
||||
<h2>存储引擎</h2>
|
||||
<p class="section-description">
|
||||
选择文件存储引擎,影响文档上传存储和文档中图片的存储方式。参数在全局设置中配置。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div v-if="loading" class="loading-inline">
|
||||
<t-loading size="small" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
|
||||
<div v-else class="settings-group">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>存储引擎</label>
|
||||
<p class="desc">选择该知识库使用的存储引擎,需在全局设置中已配置对应引擎。</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-select
|
||||
v-model="localProvider"
|
||||
size="medium"
|
||||
placeholder="请选择存储引擎"
|
||||
style="width: 100%; min-width: 220px;"
|
||||
@change="handleChange"
|
||||
>
|
||||
<t-option
|
||||
v-for="opt in engineOptions"
|
||||
:key="opt.value"
|
||||
:value="opt.value"
|
||||
:label="opt.label"
|
||||
:disabled="opt.disabled"
|
||||
>
|
||||
<span class="select-option">
|
||||
<span>{{ opt.label }}</span>
|
||||
<t-tag v-if="opt.disabled" theme="warning" variant="light" size="small">未配置</t-tag>
|
||||
<t-tag v-else-if="opt.available === false" theme="danger" variant="light" size="small">不可用</t-tag>
|
||||
</span>
|
||||
</t-option>
|
||||
</t-select>
|
||||
<p v-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
|
||||
<a v-if="showGoSettings" href="javascript:void(0)" class="go-settings" @click.prevent="goToStorageSettings">去全局设置中配置</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, watch, onMounted } from 'vue'
|
||||
import { getStorageEngineConfig, getStorageEngineStatus, type StorageEngineStatusItem } from '@/api/system'
|
||||
import { useUIStore } from '@/stores/ui'
|
||||
|
||||
const props = defineProps<{
|
||||
storageProvider: string
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:storageProvider': [value: string]
|
||||
}>()
|
||||
|
||||
const uiStore = useUIStore()
|
||||
const localProvider = ref(props.storageProvider || 'local')
|
||||
const loading = ref(true)
|
||||
const engineStatus = ref<StorageEngineStatusItem[]>([])
|
||||
const defaultProvider = ref('local')
|
||||
const hasAnyConfig = ref(false)
|
||||
|
||||
const engineOptions = computed(() => {
|
||||
const statusMap: Record<string, boolean> = {}
|
||||
for (const e of engineStatus.value) {
|
||||
statusMap[e.name] = e.available
|
||||
}
|
||||
return [
|
||||
{
|
||||
value: 'local',
|
||||
label: 'Local(本地存储)',
|
||||
desc: '仅适合单机部署,简单轻量',
|
||||
available: statusMap.local !== false,
|
||||
disabled: false,
|
||||
},
|
||||
{
|
||||
value: 'minio',
|
||||
label: 'MinIO',
|
||||
desc: 'S3 兼容,适合内网或私有云',
|
||||
available: statusMap.minio,
|
||||
disabled: statusMap.minio === false,
|
||||
},
|
||||
{
|
||||
value: 'cos',
|
||||
label: '腾讯云 COS',
|
||||
desc: '公有云部署,支持 CDN 加速',
|
||||
available: statusMap.cos,
|
||||
disabled: statusMap.cos === false,
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
const showGoSettings = computed(() =>
|
||||
engineOptions.value.some(o => o.disabled)
|
||||
)
|
||||
|
||||
const selectedOption = computed(() =>
|
||||
engineOptions.value.find(o => o.value === localProvider.value)
|
||||
)
|
||||
|
||||
function handleChange() {
|
||||
emit('update:storageProvider', localProvider.value)
|
||||
}
|
||||
|
||||
function goToStorageSettings() {
|
||||
uiStore.closeKBEditor?.()
|
||||
uiStore.openSettings?.('storage')
|
||||
}
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
const [configRes, statusRes] = await Promise.all([
|
||||
getStorageEngineConfig(),
|
||||
getStorageEngineStatus(),
|
||||
])
|
||||
const engines = statusRes?.data?.engines ?? []
|
||||
engineStatus.value = engines
|
||||
defaultProvider.value = configRes?.data?.default_provider || 'local'
|
||||
const d = configRes?.data
|
||||
hasAnyConfig.value = !!(d?.local?.path_prefix || d?.minio?.bucket_name || d?.cos?.bucket_name)
|
||||
if (!localProvider.value || localProvider.value === '') {
|
||||
localProvider.value = defaultProvider.value
|
||||
emit('update:storageProvider', localProvider.value)
|
||||
}
|
||||
} catch {
|
||||
engineStatus.value = []
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch(() => props.storageProvider, (v) => {
|
||||
localProvider.value = v || defaultProvider.value || 'local'
|
||||
}, { immediate: true })
|
||||
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.kb-storage-settings {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
margin-bottom: 32px;
|
||||
|
||||
h2 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin: 0 0 8px 0;
|
||||
}
|
||||
|
||||
.section-description {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.loading-inline {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 16px 0;
|
||||
}
|
||||
|
||||
.settings-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.setting-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
padding: 20px 0;
|
||||
border-bottom: 1px solid #e5e7eb;
|
||||
}
|
||||
|
||||
.setting-info {
|
||||
flex: 1;
|
||||
max-width: 65%;
|
||||
padding-right: 24px;
|
||||
|
||||
label {
|
||||
font-size: 15px;
|
||||
font-weight: 500;
|
||||
color: #333;
|
||||
display: block;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.desc {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.setting-control {
|
||||
flex-shrink: 0;
|
||||
min-width: 280px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.select-option {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.option-hint {
|
||||
font-size: 12px;
|
||||
color: #999;
|
||||
margin: 0;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.go-settings {
|
||||
font-size: 13px;
|
||||
color: var(--td-brand-color, #0052d9);
|
||||
margin-top: 8px;
|
||||
text-decoration: none;
|
||||
|
||||
&:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
<Menu></Menu>
|
||||
<RouterView />
|
||||
<div class="upload-mask" v-show="ismask">
|
||||
<input type="file" style="display: none" ref="uploadInput" accept=".pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xls,.xlsx" />
|
||||
<input type="file" style="display: none" ref="uploadInput" accept=".pdf,.docx,.doc,.pptx,.ppt,.txt,.md,.jpg,.jpeg,.png,.csv,.xls,.xlsx" />
|
||||
<UploadMask></UploadMask>
|
||||
</div>
|
||||
<!-- 全局设置模态框,供所有 platform 子路由使用 -->
|
||||
|
|
|
|||
|
|
@ -0,0 +1,610 @@
|
|||
<template>
|
||||
<div class="parser-engine-settings">
|
||||
<div class="section-header">
|
||||
<h2>解析引擎</h2>
|
||||
<p class="section-description">
|
||||
文档解析引擎状态及配置。此处设置优先于服务端环境变量,留空则使用环境变量默认值。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div v-if="loading" class="loading-state">
|
||||
<t-loading size="small" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
|
||||
<div v-else-if="error" class="error-inline">
|
||||
<t-alert theme="error" :message="error">
|
||||
<template #operation>
|
||||
<t-button size="small" @click="loadAll">重试</t-button>
|
||||
</template>
|
||||
</t-alert>
|
||||
</div>
|
||||
|
||||
<template v-else>
|
||||
<!-- 引擎列表 -->
|
||||
<div class="engine-section">
|
||||
<div v-if="engines.length === 0" class="empty-state">
|
||||
<p class="empty-text">未检测到解析引擎,请确认 DocReader 服务正常运行。</p>
|
||||
</div>
|
||||
|
||||
<div v-else class="engine-grid">
|
||||
<!-- DocReader 未连接时显示占位卡片 -->
|
||||
<div v-if="!hasBuiltinEngine" class="engine-card unavailable">
|
||||
<div class="card-header">
|
||||
<span class="card-name">builtin</span>
|
||||
<t-tag theme="danger" variant="light" size="small">未连接</t-tag>
|
||||
</div>
|
||||
<p class="card-desc">DocReader 内置解析引擎(docx/pdf/xlsx 等复杂格式)</p>
|
||||
<div class="docreader-inline">
|
||||
<div class="status-line">
|
||||
<t-tag theme="danger" variant="light" size="small">未连接</t-tag>
|
||||
<t-tag theme="default" variant="light" size="small">{{ docreaderTransport === 'http' ? 'HTTP' : 'gRPC' }}</t-tag>
|
||||
<span v-if="docreaderAddrEnv" class="env-hint">当前: {{ docreaderAddrEnv }}</span>
|
||||
</div>
|
||||
<p class="desc docreader-desc">
|
||||
修改请设置环境变量 <code>DOCREADER_ADDR</code>、<code>DOCREADER_TRANSPORT</code>(grpc/http),重启服务生效。
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-for="engine in sortedEngines"
|
||||
:key="engine.Name"
|
||||
:class="['engine-card', { unavailable: !engine.Available }]"
|
||||
>
|
||||
<div class="card-header">
|
||||
<span class="card-name">{{ engine.Name }}</span>
|
||||
<t-tag v-if="engine.Available" theme="success" variant="light" size="small">可用</t-tag>
|
||||
<t-tooltip v-else-if="engine.UnavailableReason" :content="engine.UnavailableReason" placement="top">
|
||||
<t-tag theme="danger" variant="light" size="small" class="tag-with-tooltip">不可用</t-tag>
|
||||
</t-tooltip>
|
||||
<t-tag v-else theme="danger" variant="light" size="small">不可用</t-tag>
|
||||
<a
|
||||
v-if="engineDocLink(engine.Name)"
|
||||
:href="engineDocLink(engine.Name)"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="card-doc-link"
|
||||
>{{ engineDocLabel(engine.Name) }} ↗</a>
|
||||
</div>
|
||||
|
||||
<p class="card-desc">{{ engine.Description }}</p>
|
||||
|
||||
<!-- builtin 引擎:内嵌 DocReader 连接信息 -->
|
||||
<div v-if="engine.Name === 'builtin'" class="docreader-inline">
|
||||
<div class="status-line">
|
||||
<t-tag v-if="connected" theme="success" variant="light" size="small">已连接</t-tag>
|
||||
<t-tag v-else theme="danger" variant="light" size="small">未连接</t-tag>
|
||||
<t-tag theme="default" variant="light" size="small">{{ docreaderTransport === 'http' ? 'HTTP' : 'gRPC' }}</t-tag>
|
||||
<span v-if="docreaderAddrEnv" class="env-hint">当前: {{ docreaderAddrEnv }}</span>
|
||||
</div>
|
||||
<p class="desc docreader-desc">
|
||||
修改请设置环境变量 <code>DOCREADER_ADDR</code>、<code>DOCREADER_TRANSPORT</code>(grpc/http),重启服务生效。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="card-types">
|
||||
<t-tag
|
||||
v-for="ft in (engine.FileTypes || [])"
|
||||
:key="ft"
|
||||
size="small"
|
||||
variant="light"
|
||||
theme="default"
|
||||
>{{ ft }}</t-tag>
|
||||
</div>
|
||||
|
||||
<!-- mineru 自建配置 -->
|
||||
<div v-if="engine.Name === 'mineru'" class="card-config">
|
||||
<div class="config-item">
|
||||
<label>自建端点</label>
|
||||
<t-input
|
||||
v-model="config.mineru_endpoint"
|
||||
size="small"
|
||||
placeholder="如 https://your-mineru.example.com"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Backend</label>
|
||||
<t-select v-model="config.mineru_model" size="small" placeholder="默认 pipeline" clearable>
|
||||
<t-option value="pipeline" label="pipeline" />
|
||||
<t-option value="vlm-auto-engine" label="vlm-auto-engine" />
|
||||
<t-option value="vlm-http-client" label="vlm-http-client" />
|
||||
<t-option value="hybrid-auto-engine" label="hybrid-auto-engine" />
|
||||
<t-option value="hybrid-http-client" label="hybrid-http-client" />
|
||||
</t-select>
|
||||
</div>
|
||||
<div class="config-toggles">
|
||||
<t-checkbox v-model="config.mineru_enable_formula">公式识别</t-checkbox>
|
||||
<t-checkbox v-model="config.mineru_enable_table">表格识别</t-checkbox>
|
||||
<t-checkbox v-model="config.mineru_enable_ocr">OCR</t-checkbox>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>语言</label>
|
||||
<t-input
|
||||
v-model="config.mineru_language"
|
||||
size="small"
|
||||
placeholder="如 ch、en、ja(默认 ch)"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- mineru_cloud 云 API 配置 -->
|
||||
<div v-if="engine.Name === 'mineru_cloud'" class="card-config">
|
||||
<div class="config-item">
|
||||
<label>API Key</label>
|
||||
<t-input
|
||||
v-model="config.mineru_api_key"
|
||||
size="small"
|
||||
type="password"
|
||||
placeholder="MinerU 云服务 API Key"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>API 地址</label>
|
||||
<t-input
|
||||
v-model="config.mineru_api_base_url"
|
||||
size="small"
|
||||
placeholder="如 https://api.mineru.net"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Model Version</label>
|
||||
<t-select v-model="config.mineru_cloud_model" size="small" placeholder="默认 pipeline" clearable>
|
||||
<t-option value="pipeline" label="pipeline" />
|
||||
<t-option value="vlm" label="vlm(视觉语言模型)" />
|
||||
<t-option value="MinerU-HTML" label="MinerU-HTML(HTML 解析)" />
|
||||
</t-select>
|
||||
</div>
|
||||
<div class="config-toggles">
|
||||
<t-checkbox v-model="config.mineru_cloud_enable_formula">公式识别</t-checkbox>
|
||||
<t-checkbox v-model="config.mineru_cloud_enable_table">表格识别</t-checkbox>
|
||||
<t-checkbox v-model="config.mineru_cloud_enable_ocr">OCR</t-checkbox>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>语言</label>
|
||||
<t-input
|
||||
v-model="config.mineru_cloud_language"
|
||||
size="small"
|
||||
placeholder="如 ch、en、ja(默认 ch)"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 检测与保存 -->
|
||||
<div class="save-bar">
|
||||
<t-button theme="default" variant="outline" :loading="checking" @click="onCheck">
|
||||
使用当前参数检测
|
||||
</t-button>
|
||||
<t-button theme="primary" :loading="saving" @click="onSave">保存配置</t-button>
|
||||
<span v-if="checkMessage" class="save-msg hint">{{ checkMessage }}</span>
|
||||
<span v-else-if="saveMessage" :class="['save-msg', saveSuccess ? 'success' : 'error']">
|
||||
{{ saveMessage }}
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, onMounted } from 'vue'
|
||||
import {
|
||||
getParserEngines,
|
||||
getParserEngineConfig,
|
||||
updateParserEngineConfig,
|
||||
checkParserEngines,
|
||||
type ParserEngineInfo,
|
||||
type ParserEngineConfig,
|
||||
} from '@/api/system'
|
||||
|
||||
const CONFIGURABLE_ENGINES = new Set(['mineru', 'mineru_cloud'])
|
||||
|
||||
/** 各解析引擎的项目/官方文档地址 */
|
||||
const ENGINE_DOC_LINKS: Record<string, { url: string; label: string }> = {
|
||||
markitdown: { url: 'https://github.com/microsoft/markitdown', label: 'Markitdown 文档' },
|
||||
mineru: { url: 'https://github.com/opendatalab/MinerU', label: 'MinerU 文档' },
|
||||
mineru_cloud: { url: 'https://mineru.net/apiManage/docs', label: 'MinerU 文档' },
|
||||
}
|
||||
|
||||
/** 解析引擎配置默认值(与 DocReader/Python 侧一致) */
|
||||
const DEFAULT_PARSER_CONFIG: ParserEngineConfig = {
|
||||
docreader_addr: '',
|
||||
docreader_transport: 'grpc',
|
||||
mineru_endpoint: '',
|
||||
mineru_api_key: '',
|
||||
mineru_api_base_url: 'https://mineru.net/api/v4',
|
||||
mineru_model: 'pipeline',
|
||||
mineru_enable_formula: true,
|
||||
mineru_enable_table: true,
|
||||
mineru_enable_ocr: true,
|
||||
mineru_language: 'ch',
|
||||
mineru_cloud_model: 'pipeline',
|
||||
mineru_cloud_enable_formula: true,
|
||||
mineru_cloud_enable_table: true,
|
||||
mineru_cloud_enable_ocr: true,
|
||||
mineru_cloud_language: 'ch',
|
||||
}
|
||||
|
||||
const engines = ref<ParserEngineInfo[]>([])
|
||||
const docreaderAddrEnv = ref('')
|
||||
const docreaderTransport = ref<'grpc' | 'http'>('grpc')
|
||||
const connected = ref(false)
|
||||
const loading = ref(true)
|
||||
const error = ref('')
|
||||
|
||||
const config = ref<ParserEngineConfig>({ ...DEFAULT_PARSER_CONFIG })
|
||||
const saving = ref(false)
|
||||
const saveMessage = ref('')
|
||||
const saveSuccess = ref(false)
|
||||
const checking = ref(false)
|
||||
const checkMessage = ref('')
|
||||
|
||||
const hasBuiltinEngine = computed(() => engines.value.some(e => e.Name === 'builtin'))
|
||||
|
||||
/** 固定展示顺序,未列出的引擎排在末尾按名称排序 */
|
||||
const ENGINE_ORDER: Record<string, number> = {
|
||||
builtin: 0,
|
||||
simple: 1,
|
||||
markitdown: 2,
|
||||
mineru: 3,
|
||||
mineru_cloud: 4,
|
||||
}
|
||||
|
||||
const sortedEngines = computed(() => {
|
||||
return [...engines.value].sort((a, b) => {
|
||||
const oa = ENGINE_ORDER[a.Name] ?? 100
|
||||
const ob = ENGINE_ORDER[b.Name] ?? 100
|
||||
if (oa !== ob) return oa - ob
|
||||
return a.Name.localeCompare(b.Name)
|
||||
})
|
||||
})
|
||||
|
||||
function hasConfigFields(engineName: string): boolean {
|
||||
return CONFIGURABLE_ENGINES.has(engineName)
|
||||
}
|
||||
|
||||
function engineDocLink(name: string): string | undefined {
|
||||
return ENGINE_DOC_LINKS[name]?.url
|
||||
}
|
||||
|
||||
function engineDocLabel(name: string): string {
|
||||
return ENGINE_DOC_LINKS[name]?.label ?? '文档'
|
||||
}
|
||||
|
||||
async function loadEngines() {
|
||||
try {
|
||||
const res = await getParserEngines()
|
||||
engines.value = res?.data ?? []
|
||||
docreaderAddrEnv.value = res?.docreader_addr ?? ''
|
||||
const t = (res?.docreader_transport ?? 'grpc').toLowerCase()
|
||||
docreaderTransport.value = t === 'http' ? 'http' : 'grpc'
|
||||
connected.value = res?.connected ?? (engines.value.length > 0)
|
||||
} catch (e: any) {
|
||||
error.value = e?.message || '加载解析引擎列表失败'
|
||||
engines.value = []
|
||||
connected.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadConfig() {
|
||||
try {
|
||||
const res = await getParserEngineConfig()
|
||||
const data = res?.data
|
||||
config.value = {
|
||||
docreader_addr: data?.docreader_addr ?? DEFAULT_PARSER_CONFIG.docreader_addr ?? '',
|
||||
docreader_transport: data?.docreader_transport ?? DEFAULT_PARSER_CONFIG.docreader_transport ?? 'grpc',
|
||||
mineru_endpoint: data?.mineru_endpoint ?? DEFAULT_PARSER_CONFIG.mineru_endpoint ?? '',
|
||||
mineru_api_key: data?.mineru_api_key ?? DEFAULT_PARSER_CONFIG.mineru_api_key ?? '',
|
||||
mineru_api_base_url: data?.mineru_api_base_url ?? DEFAULT_PARSER_CONFIG.mineru_api_base_url ?? '',
|
||||
mineru_model: data?.mineru_model ?? DEFAULT_PARSER_CONFIG.mineru_model ?? '',
|
||||
mineru_enable_formula: data?.mineru_enable_formula ?? DEFAULT_PARSER_CONFIG.mineru_enable_formula ?? true,
|
||||
mineru_enable_table: data?.mineru_enable_table ?? DEFAULT_PARSER_CONFIG.mineru_enable_table ?? true,
|
||||
mineru_enable_ocr: data?.mineru_enable_ocr ?? DEFAULT_PARSER_CONFIG.mineru_enable_ocr ?? true,
|
||||
mineru_language: data?.mineru_language ?? DEFAULT_PARSER_CONFIG.mineru_language ?? 'ch',
|
||||
mineru_cloud_model: data?.mineru_cloud_model ?? DEFAULT_PARSER_CONFIG.mineru_cloud_model ?? '',
|
||||
mineru_cloud_enable_formula: data?.mineru_cloud_enable_formula ?? DEFAULT_PARSER_CONFIG.mineru_cloud_enable_formula ?? true,
|
||||
mineru_cloud_enable_table: data?.mineru_cloud_enable_table ?? DEFAULT_PARSER_CONFIG.mineru_cloud_enable_table ?? true,
|
||||
mineru_cloud_enable_ocr: data?.mineru_cloud_enable_ocr ?? DEFAULT_PARSER_CONFIG.mineru_cloud_enable_ocr ?? true,
|
||||
mineru_cloud_language: data?.mineru_cloud_language ?? DEFAULT_PARSER_CONFIG.mineru_cloud_language ?? 'ch',
|
||||
}
|
||||
} catch {
|
||||
config.value = { ...DEFAULT_PARSER_CONFIG }
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAll() {
|
||||
loading.value = true
|
||||
error.value = ''
|
||||
await Promise.all([loadEngines(), loadConfig()])
|
||||
loading.value = false
|
||||
}
|
||||
|
||||
function buildConfigPayload(): ParserEngineConfig {
|
||||
return {
|
||||
docreader_addr: config.value.docreader_addr?.trim() ?? '',
|
||||
docreader_transport: (config.value.docreader_transport ?? 'grpc').trim() || 'grpc',
|
||||
mineru_endpoint: config.value.mineru_endpoint?.trim() ?? '',
|
||||
mineru_api_key: config.value.mineru_api_key?.trim() ?? '',
|
||||
mineru_api_base_url: config.value.mineru_api_base_url?.trim() ?? '',
|
||||
mineru_model: config.value.mineru_model?.trim() ?? '',
|
||||
mineru_enable_formula: config.value.mineru_enable_formula,
|
||||
mineru_enable_table: config.value.mineru_enable_table,
|
||||
mineru_enable_ocr: config.value.mineru_enable_ocr,
|
||||
mineru_language: config.value.mineru_language?.trim() ?? '',
|
||||
mineru_cloud_model: config.value.mineru_cloud_model?.trim() ?? '',
|
||||
mineru_cloud_enable_formula: config.value.mineru_cloud_enable_formula,
|
||||
mineru_cloud_enable_table: config.value.mineru_cloud_enable_table,
|
||||
mineru_cloud_enable_ocr: config.value.mineru_cloud_enable_ocr,
|
||||
mineru_cloud_language: config.value.mineru_cloud_language?.trim() ?? '',
|
||||
}
|
||||
}
|
||||
|
||||
async function onCheck() {
|
||||
if (!connected) {
|
||||
checkMessage.value = '请先确保 DocReader 服务已通过环境变量配置并已连接'
|
||||
return
|
||||
}
|
||||
checking.value = true
|
||||
checkMessage.value = ''
|
||||
try {
|
||||
const res = await checkParserEngines(buildConfigPayload())
|
||||
engines.value = res?.data ?? []
|
||||
checkMessage.value = '已使用当前填写参数检测,上方状态已更新'
|
||||
setTimeout(() => { checkMessage.value = '' }, 3000)
|
||||
} catch (e: any) {
|
||||
checkMessage.value = e?.message || '检测失败'
|
||||
} finally {
|
||||
checking.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onSave() {
|
||||
saving.value = true
|
||||
saveMessage.value = ''
|
||||
try {
|
||||
await updateParserEngineConfig(buildConfigPayload())
|
||||
saveSuccess.value = true
|
||||
saveMessage.value = '保存成功'
|
||||
loadEngines()
|
||||
} catch (e: any) {
|
||||
saveSuccess.value = false
|
||||
saveMessage.value = e?.message || '保存失败'
|
||||
} finally {
|
||||
saving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(loadAll)
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.parser-engine-settings {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
// ---- 页面标题 ----
|
||||
.section-header {
|
||||
margin-bottom: 32px;
|
||||
|
||||
h2 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin: 0 0 8px 0;
|
||||
}
|
||||
|
||||
.section-description {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 加载 / 错误 ----
|
||||
.loading-state {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
padding: 48px 0;
|
||||
color: #999;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.error-inline {
|
||||
padding: 16px 0;
|
||||
}
|
||||
|
||||
.docreader-inline {
|
||||
padding: 10px 12px;
|
||||
background: #f7f8fa;
|
||||
border-radius: 6px;
|
||||
|
||||
.status-line {
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
}
|
||||
|
||||
.docreader-desc {
|
||||
margin: 0;
|
||||
font-size: 12px;
|
||||
color: #888;
|
||||
line-height: 1.6;
|
||||
|
||||
code {
|
||||
padding: 1px 5px;
|
||||
font-size: 11px;
|
||||
background: #eee;
|
||||
border-radius: 3px;
|
||||
}
|
||||
}
|
||||
|
||||
.card-doc-link {
|
||||
margin-left: auto;
|
||||
font-size: 12px;
|
||||
color: #999;
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
|
||||
&:hover {
|
||||
color: var(--td-brand-color, #0052d9);
|
||||
}
|
||||
}
|
||||
|
||||
.status-line {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.env-hint {
|
||||
font-size: 12px;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
// ---- 引擎列表 ----
|
||||
.engine-section {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.engine-grid {
|
||||
columns: 2;
|
||||
column-gap: 12px;
|
||||
|
||||
@media (max-width: 860px) {
|
||||
columns: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.engine-card {
|
||||
break-inside: avoid;
|
||||
margin-bottom: 12px;
|
||||
padding: 16px;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 8px;
|
||||
background: #fff;
|
||||
transition: border-color 0.2s;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
|
||||
&:hover {
|
||||
border-color: #c0c4cc;
|
||||
}
|
||||
|
||||
&.unavailable {
|
||||
background: #fafafa;
|
||||
}
|
||||
}
|
||||
|
||||
.card-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.card-name {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
font-family: 'SF Mono', 'Monaco', 'Menlo', monospace;
|
||||
}
|
||||
|
||||
.card-desc {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.card-types {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.card-reason {
|
||||
font-size: 12px;
|
||||
color: #e34d59;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
// ---- 卡片内配置区 ----
|
||||
.card-config {
|
||||
margin-top: 4px;
|
||||
padding-top: 12px;
|
||||
border-top: 1px dashed #e5e7eb;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.config-item {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
|
||||
label {
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
color: #555;
|
||||
}
|
||||
}
|
||||
|
||||
.config-toggles {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
// ---- 空状态 ----
|
||||
.empty-state {
|
||||
padding: 48px 0;
|
||||
text-align: center;
|
||||
|
||||
.empty-text {
|
||||
font-size: 14px;
|
||||
color: #999;
|
||||
margin: 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 保存栏 ----
|
||||
.save-bar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 24px;
|
||||
padding-top: 24px;
|
||||
border-top: 1px solid #e5e7eb;
|
||||
}
|
||||
|
||||
.save-msg {
|
||||
font-size: 13px;
|
||||
|
||||
&.success {
|
||||
color: #52c41a;
|
||||
}
|
||||
|
||||
&.error {
|
||||
color: #ff4d4f;
|
||||
}
|
||||
|
||||
&.hint {
|
||||
color: #666;
|
||||
}
|
||||
}
|
||||
|
||||
.tag-with-tooltip {
|
||||
cursor: help;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -94,6 +94,16 @@
|
|||
<WebSearchSettings />
|
||||
</div>
|
||||
|
||||
<!-- 解析引擎 -->
|
||||
<div v-if="currentSection === 'parser'" class="section">
|
||||
<ParserEngineSettings />
|
||||
</div>
|
||||
|
||||
<!-- 存储引擎 -->
|
||||
<div v-if="currentSection === 'storage'" class="section">
|
||||
<StorageEngineSettings />
|
||||
</div>
|
||||
|
||||
<!-- 系统信息 -->
|
||||
<div v-if="currentSection === 'system'" class="section">
|
||||
<SystemInfo />
|
||||
|
|
@ -135,6 +145,8 @@ import ModelSettings from './ModelSettings.vue'
|
|||
import OllamaSettings from './OllamaSettings.vue'
|
||||
import McpSettings from './McpSettings.vue'
|
||||
import WebSearchSettings from './WebSearchSettings.vue'
|
||||
import ParserEngineSettings from './ParserEngineSettings.vue'
|
||||
import StorageEngineSettings from './StorageEngineSettings.vue'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
|
@ -160,6 +172,8 @@ const navItems = computed(() => [
|
|||
},
|
||||
{ key: 'ollama', icon: 'server', label: 'Ollama' },
|
||||
{ key: 'websearch', icon: 'search', label: t('settings.webSearchConfig') },
|
||||
{ key: 'parser', icon: 'file-search', label: '解析引擎' },
|
||||
{ key: 'storage', icon: 'cloud', label: t('settings.storageEngine') },
|
||||
{ key: 'mcp', icon: 'tools', label: t('settings.mcpService') },
|
||||
{ key: 'system', icon: 'info-circle', label: t('settings.systemSettings') },
|
||||
{ key: 'tenant', icon: 'user-circle', label: t('settings.tenantInfo') },
|
||||
|
|
|
|||
|
|
@ -0,0 +1,739 @@
|
|||
<template>
|
||||
<div class="storage-engine-settings">
|
||||
<div class="section-header">
|
||||
<h2>存储引擎</h2>
|
||||
<p class="section-description">
|
||||
配置文档与图片的存储方式。此处设置各引擎参数,知识库中仅选择使用哪个引擎。存储引擎会影响文档上传存储以及文档内图片的存储。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div v-if="loading" class="loading-state">
|
||||
<t-loading size="small" />
|
||||
<span>加载中...</span>
|
||||
</div>
|
||||
|
||||
<div v-else-if="error" class="error-inline">
|
||||
<t-alert theme="error" :message="error">
|
||||
<template #operation>
|
||||
<t-button size="small" @click="loadAll">重试</t-button>
|
||||
</template>
|
||||
</t-alert>
|
||||
</div>
|
||||
|
||||
<template v-else>
|
||||
<div class="engine-section">
|
||||
<div class="section-title-row">
|
||||
<div class="section-title-info">
|
||||
<h3>引擎配置</h3>
|
||||
<p>为每种存储引擎配置参数,知识库创建或编辑时选择使用哪个引擎。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="engine-list">
|
||||
<!-- Local -->
|
||||
<div class="engine-block">
|
||||
<div class="block-header">
|
||||
<span class="block-name">Local(本地存储)</span>
|
||||
<t-tag theme="success" variant="light" size="small">可用</t-tag>
|
||||
</div>
|
||||
<p class="block-desc">使用服务器本地文件系统存储文件,仅适合单机部署。</p>
|
||||
<div class="block-config">
|
||||
<div class="config-item">
|
||||
<label>路径前缀(可选)</label>
|
||||
<t-input
|
||||
v-model="config.local.path_prefix"
|
||||
size="small"
|
||||
placeholder="如 weknora/images"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- MinIO -->
|
||||
<div class="engine-block">
|
||||
<div class="block-header">
|
||||
<span class="block-name">MinIO</span>
|
||||
<t-tag v-if="minioAvailable" theme="success" variant="light" size="small">可用</t-tag>
|
||||
<t-tag v-else theme="default" variant="light" size="small">需要配置</t-tag>
|
||||
</div>
|
||||
<p class="block-desc">S3 兼容的自托管对象存储,适合内网和私有云部署。</p>
|
||||
|
||||
<div class="minio-mode-tabs">
|
||||
<div
|
||||
:class="['mode-tab', { active: config.minio.mode !== 'remote' }]"
|
||||
@click="config.minio.mode = 'docker'"
|
||||
>
|
||||
<span class="tab-label">Docker 部署</span>
|
||||
<t-tag v-if="minioEnvAvailable" theme="success" variant="light" size="small">已检测</t-tag>
|
||||
<t-tag v-else theme="default" variant="light" size="small">未检测到</t-tag>
|
||||
</div>
|
||||
<div
|
||||
:class="['mode-tab', { active: config.minio.mode === 'remote' }]"
|
||||
@click="config.minio.mode = 'remote'"
|
||||
>
|
||||
<span class="tab-label">远程 MinIO</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Docker mode -->
|
||||
<div v-if="config.minio.mode !== 'remote'" class="minio-mode-content">
|
||||
<p v-if="minioEnvAvailable" class="mode-hint success">
|
||||
已检测到 Docker 部署的 MinIO 环境变量,连接信息(Endpoint、Access Key)由环境变量提供,无需手动填写。
|
||||
</p>
|
||||
<p v-else class="mode-hint warning">
|
||||
未检测到 MinIO 环境变量(MINIO_ENDPOINT、MINIO_ACCESS_KEY_ID、MINIO_SECRET_ACCESS_KEY),请确认 Docker Compose 配置正确。
|
||||
</p>
|
||||
<div class="block-config">
|
||||
<div class="config-item">
|
||||
<label>Bucket 名称</label>
|
||||
<t-select
|
||||
v-model="config.minio.bucket_name"
|
||||
size="small"
|
||||
filterable
|
||||
creatable
|
||||
placeholder="选择或输入 Bucket"
|
||||
:loading="loadingBuckets"
|
||||
:disabled="!minioEnvAvailable"
|
||||
@focus="loadMinioBuckets"
|
||||
style="width: 100%;"
|
||||
>
|
||||
<t-option
|
||||
v-for="b in minioBuckets"
|
||||
:key="b.name"
|
||||
:value="b.name"
|
||||
:label="b.name"
|
||||
/>
|
||||
</t-select>
|
||||
</div>
|
||||
<div class="config-item config-item--inline">
|
||||
<label>Use SSL</label>
|
||||
<t-switch v-model="config.minio.use_ssl" size="small" />
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>路径前缀(可选)</label>
|
||||
<t-input
|
||||
v-model="config.minio.path_prefix"
|
||||
size="small"
|
||||
placeholder="如 weknora"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="minioEnvAvailable" class="check-bar">
|
||||
<t-button size="small" variant="outline" :loading="checkingMinio" @click="onCheckMinio">测试连接</t-button>
|
||||
<span v-if="minioCheckResult" :class="['check-msg', minioCheckResult.ok ? 'success' : 'error']">
|
||||
{{ minioCheckResult.message }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Remote mode -->
|
||||
<div v-else class="minio-mode-content">
|
||||
<p class="mode-hint">连接到远程 MinIO 服务,需要手动填写连接信息。</p>
|
||||
<div class="block-config">
|
||||
<div class="config-item">
|
||||
<label>Endpoint</label>
|
||||
<t-input
|
||||
v-model="config.minio.endpoint"
|
||||
size="small"
|
||||
placeholder="如 minio.example.com:9000"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Access Key ID</label>
|
||||
<t-input
|
||||
v-model="config.minio.access_key_id"
|
||||
size="small"
|
||||
placeholder="MinIO Access Key"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Secret Access Key</label>
|
||||
<t-input
|
||||
v-model="config.minio.secret_access_key"
|
||||
size="small"
|
||||
type="password"
|
||||
placeholder="MinIO Secret Key"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Bucket 名称</label>
|
||||
<t-input
|
||||
v-model="config.minio.bucket_name"
|
||||
size="small"
|
||||
placeholder="存储桶名称"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item config-item--inline">
|
||||
<label>Use SSL</label>
|
||||
<t-switch v-model="config.minio.use_ssl" size="small" />
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>路径前缀(可选)</label>
|
||||
<t-input
|
||||
v-model="config.minio.path_prefix"
|
||||
size="small"
|
||||
placeholder="如 weknora"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="check-bar">
|
||||
<t-button size="small" variant="outline" :loading="checkingMinio" @click="onCheckMinio">测试连接</t-button>
|
||||
<span v-if="minioCheckResult" :class="['check-msg', minioCheckResult.ok ? 'success' : 'error']">
|
||||
{{ minioCheckResult.message }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- COS -->
|
||||
<div class="engine-block">
|
||||
<div class="block-header">
|
||||
<span class="block-name">腾讯云 COS</span>
|
||||
<t-tag theme="success" variant="light" size="small">可配置</t-tag>
|
||||
</div>
|
||||
<p class="block-desc">腾讯云对象存储服务,适合公有云部署,支持 CDN 加速。</p>
|
||||
<div class="block-config">
|
||||
<div class="config-item">
|
||||
<label>Secret ID</label>
|
||||
<t-input
|
||||
v-model="config.cos.secret_id"
|
||||
size="small"
|
||||
placeholder="腾讯云 API 密钥 SecretId"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Secret Key</label>
|
||||
<t-input
|
||||
v-model="config.cos.secret_key"
|
||||
size="small"
|
||||
type="password"
|
||||
placeholder="腾讯云 API 密钥 SecretKey"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Region</label>
|
||||
<t-input
|
||||
v-model="config.cos.region"
|
||||
size="small"
|
||||
placeholder="如 ap-guangzhou"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>Bucket 名称</label>
|
||||
<t-input
|
||||
v-model="config.cos.bucket_name"
|
||||
size="small"
|
||||
placeholder="存储桶名称"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>App ID</label>
|
||||
<t-input
|
||||
v-model="config.cos.app_id"
|
||||
size="small"
|
||||
placeholder="腾讯云账号 AppID"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label>路径前缀(可选)</label>
|
||||
<t-input
|
||||
v-model="config.cos.path_prefix"
|
||||
size="small"
|
||||
placeholder="如 weknora"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="check-bar">
|
||||
<t-button size="small" variant="outline" :loading="checkingCos" @click="onCheckCos">测试连接</t-button>
|
||||
<span v-if="cosCheckResult" :class="['check-msg', cosCheckResult.ok ? 'success' : 'error']">
|
||||
{{ cosCheckResult.message }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="default-and-save">
|
||||
<div class="default-row">
|
||||
<label>默认引擎</label>
|
||||
<t-select
|
||||
v-model="config.default_provider"
|
||||
size="small"
|
||||
style="width: 200px;"
|
||||
>
|
||||
<t-option value="local" label="Local(本地)" />
|
||||
<t-option value="minio" label="MinIO" />
|
||||
<t-option value="cos" label="腾讯云 COS" />
|
||||
</t-select>
|
||||
<span class="hint">新建知识库时默认选用的存储引擎</span>
|
||||
</div>
|
||||
<div class="save-bar">
|
||||
<t-button theme="primary" :loading="saving" @click="onSave">保存配置</t-button>
|
||||
<span v-if="saveMessage" :class="['save-msg', saveSuccess ? 'success' : 'error']">
|
||||
{{ saveMessage }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, onMounted } from 'vue'
|
||||
import {
|
||||
getStorageEngineConfig,
|
||||
updateStorageEngineConfig,
|
||||
getStorageEngineStatus,
|
||||
listMinioBuckets,
|
||||
checkStorageEngine,
|
||||
type StorageEngineConfig,
|
||||
type MinioBucketInfo,
|
||||
} from '@/api/system'
|
||||
|
||||
const defaultConfig = (): StorageEngineConfig => ({
|
||||
default_provider: 'local',
|
||||
local: { path_prefix: '' },
|
||||
minio: { mode: 'docker', endpoint: '', access_key_id: '', secret_access_key: '', bucket_name: '', use_ssl: false, path_prefix: '' },
|
||||
cos: {
|
||||
secret_id: '',
|
||||
secret_key: '',
|
||||
region: '',
|
||||
bucket_name: '',
|
||||
app_id: '',
|
||||
path_prefix: '',
|
||||
},
|
||||
})
|
||||
|
||||
const loading = ref(true)
|
||||
const error = ref('')
|
||||
const config = ref<StorageEngineConfig>(defaultConfig())
|
||||
const engineStatus = ref<{ local: boolean; minio: boolean; cos: boolean }>({
|
||||
local: true,
|
||||
minio: false,
|
||||
cos: true,
|
||||
})
|
||||
const minioEnvAvailable = ref(false)
|
||||
const minioBuckets = ref<MinioBucketInfo[]>([])
|
||||
const loadingBuckets = ref(false)
|
||||
const saving = ref(false)
|
||||
const saveMessage = ref('')
|
||||
const saveSuccess = ref(false)
|
||||
|
||||
const checkingMinio = ref(false)
|
||||
const minioCheckResult = ref<{ ok: boolean; message: string } | null>(null)
|
||||
const checkingCos = ref(false)
|
||||
const cosCheckResult = ref<{ ok: boolean; message: string } | null>(null)
|
||||
|
||||
const minioAvailable = computed(() => {
|
||||
if (config.value.minio?.mode === 'remote') {
|
||||
return !!(config.value.minio.endpoint && config.value.minio.access_key_id && config.value.minio.secret_access_key)
|
||||
}
|
||||
return minioEnvAvailable.value
|
||||
})
|
||||
|
||||
async function loadConfig() {
|
||||
try {
|
||||
const res = await getStorageEngineConfig()
|
||||
const d = res?.data
|
||||
if (d) {
|
||||
config.value = {
|
||||
default_provider: d.default_provider || 'local',
|
||||
local: d.local ? { path_prefix: d.local.path_prefix || '' } : { path_prefix: '' },
|
||||
minio: d.minio
|
||||
? {
|
||||
mode: d.minio.mode || 'docker',
|
||||
endpoint: d.minio.endpoint || '',
|
||||
access_key_id: d.minio.access_key_id || '',
|
||||
secret_access_key: d.minio.secret_access_key || '',
|
||||
bucket_name: d.minio.bucket_name || '',
|
||||
use_ssl: d.minio.use_ssl ?? false,
|
||||
path_prefix: d.minio.path_prefix || '',
|
||||
}
|
||||
: defaultConfig().minio!,
|
||||
cos: d.cos
|
||||
? {
|
||||
secret_id: d.cos.secret_id || '',
|
||||
secret_key: d.cos.secret_key || '',
|
||||
region: d.cos.region || '',
|
||||
bucket_name: d.cos.bucket_name || '',
|
||||
app_id: d.cos.app_id || '',
|
||||
path_prefix: d.cos.path_prefix || '',
|
||||
}
|
||||
: defaultConfig().cos!,
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
config.value = defaultConfig()
|
||||
}
|
||||
}
|
||||
|
||||
async function loadStatus() {
|
||||
try {
|
||||
const res = await getStorageEngineStatus()
|
||||
const engines = res?.data?.engines ?? []
|
||||
const status = { local: true, minio: false, cos: true }
|
||||
for (const e of engines) {
|
||||
if (e.name === 'local') status.local = e.available
|
||||
if (e.name === 'minio') status.minio = e.available
|
||||
if (e.name === 'cos') status.cos = e.available
|
||||
}
|
||||
engineStatus.value = status
|
||||
minioEnvAvailable.value = res?.data?.minio_env_available ?? false
|
||||
} catch {
|
||||
engineStatus.value = { local: true, minio: false, cos: true }
|
||||
minioEnvAvailable.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadMinioBuckets() {
|
||||
if (!minioEnvAvailable.value || loadingBuckets.value) return
|
||||
loadingBuckets.value = true
|
||||
try {
|
||||
const res = await listMinioBuckets()
|
||||
if (res?.data?.buckets) {
|
||||
minioBuckets.value = res.data.buckets
|
||||
}
|
||||
} catch {
|
||||
minioBuckets.value = []
|
||||
} finally {
|
||||
loadingBuckets.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAll() {
|
||||
loading.value = true
|
||||
error.value = ''
|
||||
try {
|
||||
await Promise.all([loadConfig(), loadStatus()])
|
||||
if (minioEnvAvailable.value) loadMinioBuckets()
|
||||
} catch (e: unknown) {
|
||||
error.value = e instanceof Error ? e.message : '加载失败'
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function buildPayload(): StorageEngineConfig {
|
||||
const mode = config.value.minio?.mode || 'docker'
|
||||
return {
|
||||
default_provider: config.value.default_provider || 'local',
|
||||
local: { path_prefix: (config.value.local?.path_prefix || '').trim() },
|
||||
minio: {
|
||||
mode,
|
||||
endpoint: mode === 'remote' ? (config.value.minio?.endpoint || '').trim() : '',
|
||||
access_key_id: mode === 'remote' ? (config.value.minio?.access_key_id || '').trim() : '',
|
||||
secret_access_key: mode === 'remote' ? (config.value.minio?.secret_access_key || '').trim() : '',
|
||||
bucket_name: (config.value.minio?.bucket_name || '').trim(),
|
||||
use_ssl: config.value.minio?.use_ssl ?? false,
|
||||
path_prefix: (config.value.minio?.path_prefix || '').trim(),
|
||||
},
|
||||
cos: {
|
||||
secret_id: (config.value.cos?.secret_id || '').trim(),
|
||||
secret_key: (config.value.cos?.secret_key || '').trim(),
|
||||
region: (config.value.cos?.region || '').trim(),
|
||||
bucket_name: (config.value.cos?.bucket_name || '').trim(),
|
||||
app_id: (config.value.cos?.app_id || '').trim(),
|
||||
path_prefix: (config.value.cos?.path_prefix || '').trim(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function onSave() {
|
||||
saving.value = true
|
||||
saveMessage.value = ''
|
||||
try {
|
||||
await updateStorageEngineConfig(buildPayload())
|
||||
await loadStatus()
|
||||
saveSuccess.value = true
|
||||
saveMessage.value = '保存成功'
|
||||
} catch (e: unknown) {
|
||||
saveSuccess.value = false
|
||||
saveMessage.value = e instanceof Error ? e.message : '保存失败'
|
||||
} finally {
|
||||
saving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onCheckMinio() {
|
||||
checkingMinio.value = true
|
||||
minioCheckResult.value = null
|
||||
try {
|
||||
const payload = buildPayload()
|
||||
const res = await checkStorageEngine({ provider: 'minio', minio: payload.minio })
|
||||
minioCheckResult.value = res?.data ?? { ok: false, message: '未知错误' }
|
||||
} catch (e: unknown) {
|
||||
minioCheckResult.value = { ok: false, message: e instanceof Error ? e.message : '请求失败' }
|
||||
} finally {
|
||||
checkingMinio.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onCheckCos() {
|
||||
checkingCos.value = true
|
||||
cosCheckResult.value = null
|
||||
try {
|
||||
const payload = buildPayload()
|
||||
const res = await checkStorageEngine({ provider: 'cos', cos: payload.cos })
|
||||
cosCheckResult.value = res?.data ?? { ok: false, message: '未知错误' }
|
||||
} catch (e: unknown) {
|
||||
cosCheckResult.value = { ok: false, message: e instanceof Error ? e.message : '请求失败' }
|
||||
} finally {
|
||||
checkingCos.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(loadAll)
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.storage-engine-settings {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
margin-bottom: 32px;
|
||||
|
||||
h2 {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin: 0 0 8px 0;
|
||||
}
|
||||
|
||||
.section-description {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.loading-state,
|
||||
.error-inline {
|
||||
padding: 24px 0;
|
||||
}
|
||||
|
||||
.loading-state {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.engine-section {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.section-title-row {
|
||||
margin-bottom: 20px;
|
||||
|
||||
h3 {
|
||||
font-size: 17px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin: 0 0 6px 0;
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 13px;
|
||||
color: #999;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.engine-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.engine-block {
|
||||
padding: 20px;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 10px;
|
||||
background: #fff;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.block-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.block-name {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.block-desc {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin: 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.block-config {
|
||||
margin-top: 6px;
|
||||
padding-top: 14px;
|
||||
border-top: 1px dashed #e5e7eb;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
|
||||
gap: 14px 24px;
|
||||
}
|
||||
|
||||
.config-item {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
|
||||
label {
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
color: #555;
|
||||
}
|
||||
|
||||
&--inline {
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
|
||||
label {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- MinIO mode tabs ----
|
||||
.minio-mode-tabs {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.mode-tab {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 8px 16px;
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
background: #fafafa;
|
||||
|
||||
&:hover {
|
||||
border-color: #c0c4cc;
|
||||
}
|
||||
|
||||
&.active {
|
||||
border-color: var(--td-brand-color, #0052d9);
|
||||
background: var(--td-brand-color-light, #f0f5ff);
|
||||
}
|
||||
|
||||
.tab-label {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: #333;
|
||||
}
|
||||
}
|
||||
|
||||
.minio-mode-content {
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.mode-hint {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin: 0 0 4px 0;
|
||||
line-height: 1.5;
|
||||
|
||||
&.success {
|
||||
color: #52c41a;
|
||||
}
|
||||
|
||||
&.warning {
|
||||
color: #e6a23c;
|
||||
}
|
||||
}
|
||||
|
||||
.default-and-save {
|
||||
margin-top: 24px;
|
||||
padding-top: 24px;
|
||||
border-top: 1px solid #e5e7eb;
|
||||
}
|
||||
|
||||
.default-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-bottom: 16px;
|
||||
|
||||
label {
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.hint {
|
||||
font-size: 13px;
|
||||
color: #999;
|
||||
}
|
||||
}
|
||||
|
||||
.save-bar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.save-msg {
|
||||
font-size: 13px;
|
||||
|
||||
&.success {
|
||||
color: #52c41a;
|
||||
}
|
||||
|
||||
&.error {
|
||||
color: #ff4d4f;
|
||||
}
|
||||
}
|
||||
|
||||
.check-bar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-top: 12px;
|
||||
padding-top: 12px;
|
||||
border-top: 1px dashed #f0f0f0;
|
||||
}
|
||||
|
||||
.check-msg {
|
||||
font-size: 13px;
|
||||
|
||||
&.success {
|
||||
color: #52c41a;
|
||||
}
|
||||
|
||||
&.error {
|
||||
color: #ff4d4f;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
|
@ -31,6 +31,13 @@
|
|||
<div class="setting-control">
|
||||
<span class="info-value">
|
||||
{{ systemInfo?.version || $t('system.unknown') }}
|
||||
<t-tag
|
||||
v-if="systemInfo?.edition"
|
||||
:theme="systemInfo.edition === 'lite' ? 'primary' : 'default'"
|
||||
variant="light"
|
||||
size="small"
|
||||
style="margin-left: 8px;"
|
||||
>{{ systemInfo.edition === 'lite' ? 'Lite' : 'Standard' }}</t-tag>
|
||||
<span v-if="systemInfo?.commit_id" class="commit-info">
|
||||
({{ systemInfo.commit_id }})
|
||||
</span>
|
||||
|
|
|
|||
|
|
@ -22,6 +22,11 @@ export default defineConfig({
|
|||
target: 'http://localhost:8080',
|
||||
changeOrigin: true,
|
||||
secure: false,
|
||||
},
|
||||
'/files': {
|
||||
target: 'http://localhost:8080',
|
||||
changeOrigin: true,
|
||||
secure: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
11
go.mod
|
|
@ -3,7 +3,9 @@ module github.com/Tencent/WeKnora
|
|||
go 1.24.11
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/asg017/sqlite-vec-go-bindings v0.1.6
|
||||
github.com/chromedp/chromedp v0.14.2
|
||||
github.com/duckdb/duckdb-go/v2 v2.5.4
|
||||
github.com/elastic/go-elasticsearch/v7 v7.17.10
|
||||
|
|
@ -12,7 +14,7 @@ require (
|
|||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0
|
||||
github.com/golang-migrate/migrate/v4 v4.19.1
|
||||
github.com/google/jsonschema-go v0.4.2
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hibiken/asynq v0.25.1
|
||||
|
|
@ -51,13 +53,15 @@ require (
|
|||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/driver/postgres v1.5.11
|
||||
gorm.io/gorm v1.25.12
|
||||
gorm.io/driver/sqlite v1.6.0
|
||||
gorm.io/gorm v1.30.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.18.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/JohannesKaufmann/dom v0.2.0 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
|
|
@ -133,8 +137,6 @@ require (
|
|||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/invopop/jsonschema v0.13.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
|
|
@ -155,6 +157,7 @@ require (
|
|||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.22 // indirect
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.8-0.20251223041313-25746c47c1a7 // indirect
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.7-0.20251201120310-af64f2acba38 // indirect
|
||||
github.com/minio/crc64nvme v1.0.1 // indirect
|
||||
|
|
|
|||
28
go.sum
|
|
@ -1339,6 +1339,10 @@ github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMd
|
|||
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k=
|
||||
github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ=
|
||||
github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 h1:mklaPbT4f/EiDr1Q+zPrEt9lgKAkVrIBtWf33d9GpVA=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0/go.mod h1:D56Cl9r8M5i3UwAchE+LlLc5hPN3kJtdZNVJn06lSHU=
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
|
||||
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
|
|
@ -1393,6 +1397,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
|
|||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww=
|
||||
github.com/asg017/sqlite-vec-go-bindings v0.1.6/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q=
|
||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
|
|
@ -1721,8 +1727,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
|
|||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
|
||||
|
|
@ -1896,14 +1902,10 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVT
|
|||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
|
|
@ -2063,6 +2065,8 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
|
|||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
|
|
@ -2253,7 +2257,11 @@ github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoH
|
|||
github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
|
||||
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
|
||||
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
|
||||
github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
|
|
@ -2416,6 +2424,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
|
|||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
|
||||
github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
|
|
@ -3577,8 +3587,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314=
|
||||
gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
|
||||
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
|
||||
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
|
||||
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
|
||||
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
|
||||
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||
gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -470,9 +470,10 @@ func (t *KnowledgeSearchTool) concurrentSearchByTargets(
|
|||
return
|
||||
}
|
||||
|
||||
// Wrap results with metadata
|
||||
// Wrap results with metadata and write back KB ID
|
||||
mu.Lock()
|
||||
for _, r := range kbResults {
|
||||
r.KnowledgeBaseID = st.KnowledgeBaseID
|
||||
allResults = append(allResults, &searchResultWithMeta{
|
||||
SearchResult: r,
|
||||
SourceQuery: q,
|
||||
|
|
|
|||
|
|
@ -246,11 +246,11 @@ func (r *chunkRepository) ListChunkByParentID(
|
|||
return chunks, nil
|
||||
}
|
||||
|
||||
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields.
|
||||
// Note: This will update all fields including metadata and content_hash.
|
||||
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields
|
||||
// except SeqID (auto-increment, must not be overwritten).
|
||||
// Make sure the chunk object is complete (e.g., fetched from DB) before calling this method.
|
||||
func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error {
|
||||
return r.db.WithContext(ctx).Save(chunk).Error
|
||||
return r.db.WithContext(ctx).Omit("SeqID").Save(chunk).Error
|
||||
}
|
||||
|
||||
// UpdateChunks updates chunks in batch using raw SQL for efficiency.
|
||||
|
|
@ -332,23 +332,46 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
|
|||
args = append(args, id)
|
||||
}
|
||||
|
||||
sql := fmt.Sprintf(`
|
||||
UPDATE chunks SET
|
||||
content = CASE %s END,
|
||||
is_enabled = (CASE %s END)::boolean,
|
||||
tag_id = CASE %s END,
|
||||
flags = (CASE %s END)::integer,
|
||||
status = (CASE %s END)::integer,
|
||||
updated_at = NOW()
|
||||
WHERE id IN (%s)
|
||||
`,
|
||||
strings.Join(contentCases, " "),
|
||||
strings.Join(isEnabledCases, " "),
|
||||
strings.Join(tagIDCases, " "),
|
||||
strings.Join(flagsCases, " "),
|
||||
strings.Join(statusCases, " "),
|
||||
strings.Join(inPlaceholders, ","),
|
||||
)
|
||||
isPostgres := r.db.Dialector.Name() == "postgres"
|
||||
|
||||
var sql string
|
||||
if isPostgres {
|
||||
sql = fmt.Sprintf(`
|
||||
UPDATE chunks SET
|
||||
content = CASE %s END,
|
||||
is_enabled = (CASE %s END)::boolean,
|
||||
tag_id = CASE %s END,
|
||||
flags = (CASE %s END)::integer,
|
||||
status = (CASE %s END)::integer,
|
||||
updated_at = NOW()
|
||||
WHERE id IN (%s)
|
||||
`,
|
||||
strings.Join(contentCases, " "),
|
||||
strings.Join(isEnabledCases, " "),
|
||||
strings.Join(tagIDCases, " "),
|
||||
strings.Join(flagsCases, " "),
|
||||
strings.Join(statusCases, " "),
|
||||
strings.Join(inPlaceholders, ","),
|
||||
)
|
||||
} else {
|
||||
sql = fmt.Sprintf(`
|
||||
UPDATE chunks SET
|
||||
content = CASE %s END,
|
||||
is_enabled = CASE %s END,
|
||||
tag_id = CASE %s END,
|
||||
flags = CASE %s END,
|
||||
status = CASE %s END,
|
||||
updated_at = datetime('now')
|
||||
WHERE id IN (%s)
|
||||
`,
|
||||
strings.Join(contentCases, " "),
|
||||
strings.Join(isEnabledCases, " "),
|
||||
strings.Join(tagIDCases, " "),
|
||||
strings.Join(flagsCases, " "),
|
||||
strings.Join(statusCases, " "),
|
||||
strings.Join(inPlaceholders, ","),
|
||||
)
|
||||
}
|
||||
|
||||
return r.db.WithContext(ctx).Exec(sql, args...).Error
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,613 @@
|
|||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// sqliteEmbedding stores metadata alongside the vec0 virtual table rows
|
||||
type sqliteEmbedding struct {
|
||||
ID uint `gorm:"primarykey;autoIncrement"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
SourceID string `gorm:"column:source_id;not null;uniqueIndex:idx_sqlite_emb_source"`
|
||||
SourceType int `gorm:"column:source_type;not null;uniqueIndex:idx_sqlite_emb_source"`
|
||||
ChunkID string `gorm:"column:chunk_id;index"`
|
||||
KnowledgeID string `gorm:"column:knowledge_id;index"`
|
||||
KnowledgeBaseID string `gorm:"column:knowledge_base_id;index"`
|
||||
TagID string `gorm:"column:tag_id;index"`
|
||||
Content string `gorm:"column:content;not null"`
|
||||
Dimension int `gorm:"column:dimension;not null"`
|
||||
IsEnabled *bool `gorm:"column:is_enabled;default:true;index"`
|
||||
}
|
||||
|
||||
func (sqliteEmbedding) TableName() string { return "lite_embeddings" }
|
||||
|
||||
type sqliteRepository struct {
|
||||
db *gorm.DB
|
||||
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
|
||||
}
|
||||
|
||||
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
|
||||
logger.GetLogger(context.Background()).Info("[SQLite] Initializing SQLite retriever engine repository with sqlite-vec")
|
||||
|
||||
if err := db.AutoMigrate(&sqliteEmbedding{}); err != nil {
|
||||
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
|
||||
}
|
||||
|
||||
initFTS5(db)
|
||||
|
||||
repo := &sqliteRepository{
|
||||
db: db,
|
||||
vecTables: make(map[int]bool),
|
||||
}
|
||||
|
||||
repo.ensureExistingVecTables()
|
||||
|
||||
return repo
|
||||
}
|
||||
|
||||
func initFTS5(db *gorm.DB) {
|
||||
sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
|
||||
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
|
||||
content='lite_embeddings', content_rowid='id',
|
||||
tokenize='unicode61'
|
||||
)`
|
||||
if err := db.Exec(sql).Error; err != nil {
|
||||
logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func vecTableName(dim int) string {
|
||||
return fmt.Sprintf("vec_embeddings_%d", dim)
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) ensureVecTable(dim int) {
|
||||
if dim <= 0 || r.vecTables[dim] {
|
||||
return
|
||||
}
|
||||
tbl := vecTableName(dim)
|
||||
createSQL := fmt.Sprintf(
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS %s USING vec0(embedding float[%d] distance_metric=cosine)`,
|
||||
tbl, dim,
|
||||
)
|
||||
if err := r.db.Exec(createSQL).Error; err != nil {
|
||||
if strings.Contains(err.Error(), "already exists") {
|
||||
r.vecTables[dim] = true
|
||||
return
|
||||
}
|
||||
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
|
||||
return
|
||||
}
|
||||
r.vecTables[dim] = true
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) ensureExistingVecTables() {
|
||||
var dims []int
|
||||
r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
|
||||
for _, dim := range dims {
|
||||
r.ensureVecTable(dim)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) EngineType() types.RetrieverEngineType {
|
||||
return types.SQLiteRetrieverEngineType
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) Support() []types.RetrieverType {
|
||||
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
|
||||
row := toSQLiteEmbedding(indexInfo)
|
||||
emb := extractEmbedding(params, indexInfo.SourceID)
|
||||
if len(emb) > 0 {
|
||||
row.Dimension = len(emb)
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
r.syncFTS5Insert(ctx, row)
|
||||
if len(emb) > 0 && row.ID > 0 {
|
||||
r.insertVec(ctx, row.ID, row.Dimension, emb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error {
|
||||
if len(indexInfoList) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([]*sqliteEmbedding, len(indexInfoList))
|
||||
embs := make([][]float32, len(indexInfoList))
|
||||
for i, info := range indexInfoList {
|
||||
rows[i] = toSQLiteEmbedding(info)
|
||||
emb := extractEmbedding(params, info.SourceID)
|
||||
embs[i] = emb
|
||||
if len(emb) > 0 {
|
||||
rows[i].Dimension = len(emb)
|
||||
}
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i, row := range rows {
|
||||
r.syncFTS5Insert(ctx, row)
|
||||
if len(embs[i]) > 0 && row.ID > 0 {
|
||||
r.insertVec(ctx, row.ID, row.Dimension, embs[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList []*types.IndexInfo, _ map[string]any) int64 {
|
||||
var total int64
|
||||
for _, info := range indexInfoList {
|
||||
total += int64(len(info.Content)) + 200
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
|
||||
var rows []sqliteEmbedding
|
||||
r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
|
||||
r.deleteRowsAndVecs(ctx, rows)
|
||||
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
|
||||
var rows []sqliteEmbedding
|
||||
r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
|
||||
r.deleteRowsAndVecs(ctx, rows)
|
||||
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
|
||||
var rows []sqliteEmbedding
|
||||
r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
|
||||
r.deleteRowsAndVecs(ctx, rows)
|
||||
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) CopyIndices(ctx context.Context,
|
||||
_ string,
|
||||
sourceToTargetKBIDMap map[string]string,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
_ int, _ string,
|
||||
) error {
|
||||
for sourceChunkID, targetChunkID := range sourceToTargetChunkIDMap {
|
||||
var src sqliteEmbedding
|
||||
if err := r.db.WithContext(ctx).Where("chunk_id = ?", sourceChunkID).First(&src).Error; err != nil {
|
||||
continue
|
||||
}
|
||||
newRow := sqliteEmbedding{
|
||||
SourceID: uuid.New().String(),
|
||||
SourceType: src.SourceType,
|
||||
ChunkID: targetChunkID,
|
||||
KnowledgeID: sourceToTargetKBIDMap[src.KnowledgeID],
|
||||
KnowledgeBaseID: targetKnowledgeBaseID,
|
||||
TagID: src.TagID,
|
||||
Content: src.Content,
|
||||
Dimension: src.Dimension,
|
||||
IsEnabled: src.IsEnabled,
|
||||
}
|
||||
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
|
||||
logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy chunk %s: %v", sourceChunkID, err)
|
||||
continue
|
||||
}
|
||||
r.syncFTS5Insert(ctx, &newRow)
|
||||
if src.Dimension > 0 && newRow.ID > 0 {
|
||||
r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
||||
for chunkID, enabled := range chunkStatusMap {
|
||||
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {
|
||||
for chunkID, tagID := range chunkTagMap {
|
||||
r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Retrieve ---
|
||||
|
||||
func (r *sqliteRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
var results []*types.RetrieveResult
|
||||
|
||||
if params.RetrieverType == types.KeywordsRetrieverType || params.RetrieverType == "" {
|
||||
res, err := r.keywordsRetrieve(ctx, params)
|
||||
if err != nil {
|
||||
results = append(results, &types.RetrieveResult{
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
Error: err,
|
||||
})
|
||||
} else {
|
||||
results = append(results, res...)
|
||||
}
|
||||
}
|
||||
|
||||
if params.RetrieverType == types.VectorRetrieverType || params.RetrieverType == "" {
|
||||
res, err := r.vectorRetrieve(ctx, params)
|
||||
if err != nil {
|
||||
results = append(results, &types.RetrieveResult{
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
Error: err,
|
||||
})
|
||||
} else {
|
||||
results = append(results, res...)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// --- Keywords retrieval via FTS5 ---
|
||||
func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
if params.Query == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ftsQuery := sanitizeFTS5Query(params.Query)
|
||||
|
||||
sql := `
|
||||
SELECT e.id, e.source_id, e.source_type, e.chunk_id,
|
||||
e.knowledge_id, e.knowledge_base_id, e.tag_id,
|
||||
e.content,
|
||||
bm25(lite_embeddings_fts) AS score
|
||||
FROM lite_embeddings_fts
|
||||
JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid
|
||||
WHERE lite_embeddings_fts MATCH ?
|
||||
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
|
||||
`
|
||||
|
||||
args := []interface{}{ftsQuery}
|
||||
|
||||
for _, wp := range buildFilterWhere(params) {
|
||||
sql += " AND " + wp.clause
|
||||
args = append(args, wp.args...)
|
||||
}
|
||||
|
||||
sql += " ORDER BY score ASC LIMIT ?"
|
||||
args = append(args, params.TopK)
|
||||
|
||||
type ftsResult struct {
|
||||
ID uint
|
||||
SourceID string
|
||||
SourceType int
|
||||
ChunkID string
|
||||
KnowledgeID string
|
||||
KnowledgeBaseID string
|
||||
TagID string
|
||||
Content string
|
||||
Score float64
|
||||
}
|
||||
|
||||
var rows []ftsResult
|
||||
if err := r.db.WithContext(ctx).Raw(sql, args...).Scan(&rows).Error; err != nil {
|
||||
return nil, fmt.Errorf("FTS5 query failed: %w", err)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: query=%q, ftsQuery=%q, matched=%d rows", params.Query, ftsQuery, len(rows))
|
||||
|
||||
items := make([]*types.IndexWithScore, len(rows))
|
||||
for i, row := range rows {
|
||||
|
||||
// bm25 越小越相关 → 转成正向分数
|
||||
score := -row.Score
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s",
|
||||
i+1, row.ChunkID, row.Score, score, row.Content)
|
||||
|
||||
items[i] = &types.IndexWithScore{
|
||||
ID: fmt.Sprintf("%d", row.ID),
|
||||
SourceID: row.SourceID,
|
||||
SourceType: types.SourceType(row.SourceType),
|
||||
ChunkID: row.ChunkID,
|
||||
KnowledgeID: row.KnowledgeID,
|
||||
KnowledgeBaseID: row.KnowledgeBaseID,
|
||||
TagID: row.TagID,
|
||||
Content: row.Content,
|
||||
Score: score,
|
||||
MatchType: types.MatchTypeKeywords,
|
||||
}
|
||||
}
|
||||
|
||||
return []*types.RetrieveResult{{
|
||||
Results: items,
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
}}, nil
|
||||
}
|
||||
func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
if len(params.Embedding) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
dim := len(params.Embedding)
|
||||
r.ensureVecTable(dim)
|
||||
|
||||
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize query vector failed: %w", err)
|
||||
}
|
||||
|
||||
tbl := vecTableName(dim)
|
||||
|
||||
// ⚠️ sqlite-vec 要求必须有 k = ?
|
||||
vecSQL := fmt.Sprintf(`
|
||||
SELECT v.rowid, v.distance,
|
||||
e.source_id, e.source_type, e.chunk_id,
|
||||
e.knowledge_id, e.knowledge_base_id,
|
||||
e.tag_id, e.content
|
||||
FROM %s v
|
||||
JOIN lite_embeddings e ON e.id = v.rowid
|
||||
WHERE v.embedding MATCH ?
|
||||
AND k = ?
|
||||
AND (e.is_enabled IS NULL OR e.is_enabled = 1)
|
||||
`, tbl)
|
||||
|
||||
args := []interface{}{
|
||||
queryBlob,
|
||||
params.TopK, // 这里就是 k
|
||||
}
|
||||
|
||||
// 追加过滤条件
|
||||
for _, wp := range buildFilterWhere(params) {
|
||||
vecSQL += " AND " + wp.clause
|
||||
args = append(args, wp.args...)
|
||||
}
|
||||
|
||||
// ⚠️ 这里仍然建议加 ORDER BY,虽然 vec0 已经按距离返回
|
||||
vecSQL += " ORDER BY v.distance ASC"
|
||||
|
||||
type row struct {
|
||||
Rowid uint
|
||||
Distance float64
|
||||
SourceID string
|
||||
SourceType int
|
||||
ChunkID string
|
||||
KnowledgeID string
|
||||
KnowledgeBaseID string
|
||||
TagID string
|
||||
Content string
|
||||
}
|
||||
|
||||
var rows []row
|
||||
if err := r.db.WithContext(ctx).
|
||||
Raw(vecSQL, args...).
|
||||
Scan(&rows).Error; err != nil {
|
||||
return nil, fmt.Errorf("sqlite-vec query failed: %w", err)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: query_dim=%d, threshold=%.4f, matched=%d rows", dim, params.Threshold, len(rows))
|
||||
|
||||
items := make([]*types.IndexWithScore, 0, len(rows))
|
||||
|
||||
for i, v := range rows {
|
||||
// cosine distance = 1 - cosine_similarity
|
||||
score := 1 - v.Distance
|
||||
|
||||
logger.GetLogger(ctx).Infof("[SQLite] vectorRetrieve: #%d chunk_id=%s, distance=%.4f, score=%.4f, content_preview=%.60s",
|
||||
i+1, v.ChunkID, v.Distance, score, v.Content)
|
||||
|
||||
items = append(items, &types.IndexWithScore{
|
||||
ID: fmt.Sprintf("%d", v.Rowid),
|
||||
SourceID: v.SourceID,
|
||||
SourceType: types.SourceType(v.SourceType),
|
||||
ChunkID: v.ChunkID,
|
||||
KnowledgeID: v.KnowledgeID,
|
||||
KnowledgeBaseID: v.KnowledgeBaseID,
|
||||
TagID: v.TagID,
|
||||
Content: v.Content,
|
||||
Score: score,
|
||||
MatchType: types.MatchTypeEmbedding,
|
||||
})
|
||||
}
|
||||
|
||||
return []*types.RetrieveResult{{
|
||||
Results: items,
|
||||
RetrieverEngineType: types.SQLiteRetrieverEngineType,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// --- Internal helpers ---
|
||||
|
||||
func toSQLiteEmbedding(info *types.IndexInfo) *sqliteEmbedding {
|
||||
enabled := true
|
||||
return &sqliteEmbedding{
|
||||
SourceID: info.SourceID,
|
||||
SourceType: int(info.SourceType),
|
||||
ChunkID: info.ChunkID,
|
||||
KnowledgeID: info.KnowledgeID,
|
||||
KnowledgeBaseID: info.KnowledgeBaseID,
|
||||
TagID: info.TagID,
|
||||
Content: common.CleanInvalidUTF8(info.Content),
|
||||
Dimension: 0,
|
||||
IsEnabled: &enabled,
|
||||
}
|
||||
}
|
||||
|
||||
func extractEmbedding(params map[string]any, sourceID string) []float32 {
|
||||
if params == nil {
|
||||
return nil
|
||||
}
|
||||
embMap, ok := params["embedding"].(map[string][]float32)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return embMap[sourceID]
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) insertVec(_ context.Context, rowID uint, dim int, emb []float32) {
|
||||
r.ensureVecTable(dim)
|
||||
blob, err := sqlite_vec.SerializeFloat32(emb)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
|
||||
r.db.Exec(sql, rowID, blob)
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmbedding) {
|
||||
dimIDs := make(map[int][]uint)
|
||||
for _, row := range rows {
|
||||
if row.Dimension > 0 {
|
||||
dimIDs[row.Dimension] = append(dimIDs[row.Dimension], row.ID)
|
||||
}
|
||||
}
|
||||
for dim, ids := range dimIDs {
|
||||
if !r.vecTables[dim] {
|
||||
continue
|
||||
}
|
||||
tbl := vecTableName(dim)
|
||||
for _, id := range ids {
|
||||
r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
|
||||
if !r.vecTables[dim] {
|
||||
return
|
||||
}
|
||||
tbl := vecTableName(dim)
|
||||
r.db.Exec(fmt.Sprintf(
|
||||
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
|
||||
tbl, tbl,
|
||||
), dstID, srcID)
|
||||
}
|
||||
|
||||
func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbedding) {
|
||||
if row.ID == 0 {
|
||||
return
|
||||
}
|
||||
sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
|
||||
r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
|
||||
}
|
||||
|
||||
type whereClause struct {
|
||||
clause string
|
||||
args []interface{}
|
||||
}
|
||||
|
||||
func buildFilterWhere(params types.RetrieveParams) []whereClause {
|
||||
var parts []whereClause
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
parts = append(parts, whereClause{
|
||||
clause: "e.knowledge_base_id IN (" + placeholders(len(params.KnowledgeBaseIDs)) + ")",
|
||||
args: toInterfaceSlice(params.KnowledgeBaseIDs),
|
||||
})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
parts = append(parts, whereClause{
|
||||
clause: "e.knowledge_id IN (" + placeholders(len(params.KnowledgeIDs)) + ")",
|
||||
args: toInterfaceSlice(params.KnowledgeIDs),
|
||||
})
|
||||
}
|
||||
if len(params.TagIDs) > 0 {
|
||||
parts = append(parts, whereClause{
|
||||
clause: "e.tag_id IN (" + placeholders(len(params.TagIDs)) + ")",
|
||||
args: toInterfaceSlice(params.TagIDs),
|
||||
})
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func placeholders(n int) string {
|
||||
p := make([]string, n)
|
||||
for i := range p {
|
||||
p[i] = "?"
|
||||
}
|
||||
return strings.Join(p, ",")
|
||||
}
|
||||
|
||||
func toInterfaceSlice(ss []string) []interface{} {
|
||||
out := make([]interface{}, len(ss))
|
||||
for i, s := range ss {
|
||||
out[i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sanitizeFTS5Query builds an FTS5 query from user input.
|
||||
// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used
|
||||
// by Elasticsearch etc.) and joined with OR for broad partial matching.
|
||||
// Non-CJK words are kept intact. BM25 ranking naturally boosts documents
|
||||
// that match more tokens.
|
||||
func sanitizeFTS5Query(q string) string {
|
||||
q = strings.TrimSpace(q)
|
||||
if q == "" {
|
||||
return q
|
||||
}
|
||||
|
||||
var cjkRunes []rune
|
||||
var nonCJKWords []string
|
||||
var buf strings.Builder
|
||||
|
||||
flushNonCJK := func() {
|
||||
if buf.Len() > 0 {
|
||||
nonCJKWords = append(nonCJKWords, buf.String())
|
||||
buf.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range q {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
flushNonCJK()
|
||||
cjkRunes = append(cjkRunes, r)
|
||||
} else if unicode.IsSpace(r) || r == '|' {
|
||||
flushNonCJK()
|
||||
} else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' {
|
||||
// skip FTS5 special characters
|
||||
} else {
|
||||
buf.WriteRune(r)
|
||||
}
|
||||
}
|
||||
flushNonCJK()
|
||||
|
||||
var parts []string
|
||||
|
||||
// CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...)
|
||||
if len(cjkRunes) == 1 {
|
||||
parts = append(parts, `"`+string(cjkRunes[0])+`"`)
|
||||
} else {
|
||||
for i := 0; i < len(cjkRunes)-1; i++ {
|
||||
bigram := string(cjkRunes[i]) + string(cjkRunes[i+1])
|
||||
parts = append(parts, `"`+bigram+`"`)
|
||||
}
|
||||
}
|
||||
|
||||
for _, w := range nonCJKWords {
|
||||
w = strings.ReplaceAll(w, `"`, `""`)
|
||||
parts = append(parts, `"`+w+`"`)
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return `"` + strings.ReplaceAll(q, `"`, `""`) + `"`
|
||||
}
|
||||
return strings.Join(parts, " OR ")
|
||||
}
|
||||
|
|
@ -114,7 +114,7 @@ func (r *tenantRepository) AdjustStorageUsed(ctx context.Context, tenantID uint6
|
|||
tenant.StorageUsed += delta
|
||||
// 保存更新并验证业务规则
|
||||
if tenant.StorageUsed < 0 {
|
||||
logger.Error(ctx, "tenant storage used is negative %s: %d", tenant.ID, tenant.StorageUsed)
|
||||
logger.Errorf(ctx, "tenant storage used is negative %d: %d", tenant.ID, tenant.StorageUsed)
|
||||
tenant.StorageUsed = 0
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -183,6 +183,9 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
|||
return
|
||||
}
|
||||
if len(res) > 0 {
|
||||
for _, r := range res {
|
||||
r.KnowledgeBaseID = t.KnowledgeBaseID
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"query": q,
|
||||
|
|
@ -338,6 +341,9 @@ func (p *PluginSearch) searchByTargets(
|
|||
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
|
||||
|
||||
if len(directResults) > 0 {
|
||||
for _, r := range directResults {
|
||||
r.KnowledgeBaseID = t.KnowledgeBaseID
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"loaded_count": len(directResults),
|
||||
|
|
@ -383,6 +389,9 @@ func (p *PluginSearch) searchByTargets(
|
|||
})
|
||||
return
|
||||
}
|
||||
for _, r := range res {
|
||||
r.KnowledgeBaseID = t.KnowledgeBaseID
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"target_type": t.Type,
|
||||
|
|
|
|||
|
|
@ -222,5 +222,6 @@ func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.S
|
|||
KnowledgeFilename: knowledge.FileName,
|
||||
KnowledgeSource: knowledge.Source,
|
||||
ChunkMetadata: chunk.Metadata,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ const (
|
|||
// NewChunkExtractTask creates a new chunk extract task
|
||||
func NewChunkExtractTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
client interfaces.TaskEnqueuer,
|
||||
tenantID uint64,
|
||||
chunkID string,
|
||||
modelID string,
|
||||
|
|
@ -109,7 +109,7 @@ func NewChunkExtractTask(
|
|||
// NewTableExtractTask creates a new table extract task
|
||||
func NewDataTableSummaryTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
client interfaces.TaskEnqueuer,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
summaryModel string,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
|
|
@ -23,11 +24,28 @@ type cosFileService struct {
|
|||
client *cos.Client
|
||||
bucketURL string
|
||||
cosPathPrefix string
|
||||
// 临时桶配置(用于存放自动过期的临时文件)
|
||||
tempClient *cos.Client
|
||||
tempBucketURL string
|
||||
}
|
||||
|
||||
// newCosClient creates a bare cosFileService with just the SDK client initialised.
|
||||
// Shared by NewCosFileService* constructors and CheckCosConnectivity.
|
||||
func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
logger.Infof(context.Background(), "newCosClient: bucketURL: %s", bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
}
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
return &cosFileService{client: client, bucketURL: bucketURL}, nil
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
|
||||
|
|
@ -35,49 +53,51 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
|
|||
|
||||
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
|
||||
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
svc, err := newCosClient(bucketName, region, secretId, secretKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
b := &cos.BaseURL{BucketURL: u}
|
||||
client := cos.NewClient(b, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.cosPathPrefix = cosPathPrefix
|
||||
|
||||
svc := &cosFileService{
|
||||
client: client,
|
||||
bucketURL: bucketURL,
|
||||
cosPathPrefix: cosPathPrefix,
|
||||
}
|
||||
|
||||
// 如果配置了临时桶,初始化临时桶客户端
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region // 默认使用相同的 region
|
||||
tempRegion = region
|
||||
}
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", tempBucketName, tempRegion)
|
||||
tempU, err := url.Parse(tempBucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
|
||||
}
|
||||
tempB := &cos.BaseURL{BucketURL: tempU}
|
||||
tempClient := cos.NewClient(tempB, &http.Client{
|
||||
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.tempClient = tempClient
|
||||
svc.tempBucketURL = tempBucketURL
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies COS is reachable by performing a HEAD request on the bucket.
|
||||
func (s *cosFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := s.client.Bucket.Head(checkCtx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckCosConnectivity tests COS connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckCosConnectivity(ctx context.Context, bucketName, region, secretID, secretKey string) error {
|
||||
svc, err := newCosClient(bucketName, region, secretID, secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// SaveFile saves a file to COS storage
|
||||
// It generates a unique name for the file and organizes it by tenant and knowledge ID
|
||||
func (s *cosFileService) SaveFile(ctx context.Context,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,11 @@ import (
|
|||
// used for testing or when file storage is not required
|
||||
type DummyFileService struct{}
|
||||
|
||||
// CheckConnectivity always succeeds for the dummy service.
|
||||
func (s *DummyFileService) CheckConnectivity(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewDummyFileService creates a new instance of DummyFileService
|
||||
func NewDummyFileService() interfaces.FileService {
|
||||
return &DummyFileService{}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,18 @@ type localFileService struct {
|
|||
baseDir string // Base directory for file storage
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies the local storage directory exists and is accessible.
|
||||
func (s *localFileService) CheckConnectivity(ctx context.Context) error {
|
||||
info, err := os.Stat(s.baseDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage directory not accessible: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("storage path is not a directory: %s", s.baseDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewLocalFileService creates a new local file service instance
|
||||
func NewLocalFileService(baseDir string) interfaces.FileService {
|
||||
return &localFileService{
|
||||
|
|
|
|||
|
|
@ -22,11 +22,10 @@ type minioFileService struct {
|
|||
bucketName string
|
||||
}
|
||||
|
||||
// NewMinioFileService creates a MinIO file service
|
||||
func NewMinioFileService(endpoint,
|
||||
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
|
||||
) (interfaces.FileService, error) {
|
||||
// Initialize MinIO client
|
||||
// newMinioClient creates a bare minioFileService with just the SDK client initialised.
|
||||
// Shared by NewMinioFileService (which also ensures the bucket exists) and
|
||||
// CheckMinioConnectivity (read-only probe).
|
||||
func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) {
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
|
|
@ -34,24 +33,60 @@ func NewMinioFileService(endpoint,
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
|
||||
}
|
||||
return &minioFileService{client: client, bucketName: bucketName}, nil
|
||||
}
|
||||
|
||||
// Check if bucket exists, create if not
|
||||
exists, err := client.BucketExists(context.Background(), bucketName)
|
||||
// NewMinioFileService creates a MinIO file service.
|
||||
// It verifies that the bucket exists and creates it if missing.
|
||||
func NewMinioFileService(endpoint,
|
||||
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
|
||||
) (interfaces.FileService, error) {
|
||||
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
exists, err := svc.client.BucketExists(context.Background(), bucketName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
err = client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{})
|
||||
if err != nil {
|
||||
if err = svc.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &minioFileService{
|
||||
client: client,
|
||||
bucketName: bucketName,
|
||||
}, nil
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies MinIO is reachable and, if a bucket is configured,
|
||||
// that the bucket exists. This is a read-only probe — it never creates a bucket.
|
||||
func (s *minioFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if s.bucketName != "" {
|
||||
exists, err := s.client.BucketExists(checkCtx, s.bucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := s.client.ListBuckets(checkCtx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckMinioConnectivity tests MinIO connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckMinioConnectivity(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) error {
|
||||
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// SaveFile saves a file to MinIO
|
||||
|
|
|
|||
|
|
@ -0,0 +1,365 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
const (
|
||||
vlmOCRPrompt = "请提取这张文档图片中的所有正文内容,用纯 Markdown 格式输出。要求:\n" +
|
||||
"1. 忽略页眉、页脚\n" +
|
||||
"2. 表格使用 Markdown 表格语法\n" +
|
||||
"3. 公式使用 LaTeX 格式(用 $ 或 $$ 包裹)\n" +
|
||||
"4. 按照原文阅读顺序组织\n" +
|
||||
"5. 只输出提取到的文本内容,不要添加任何 HTML 标签\n" +
|
||||
"如果图片中没有可识别的文字内容,请回复:无文字内容。"
|
||||
vlmCaptionPrompt = "简单凝炼的描述图片的主要内容"
|
||||
)
|
||||
|
||||
// ImageMultimodalService handles image:multimodal asynq tasks.
|
||||
// It reads images from local storage, performs OCR and VLM caption,
|
||||
// and creates child chunks for the processed content.
|
||||
type ImageMultimodalService struct {
|
||||
chunkService interfaces.ChunkService
|
||||
modelService interfaces.ModelService
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
tenantRepo interfaces.TenantRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ollamaService *ollama.OllamaService
|
||||
}
|
||||
|
||||
func NewImageMultimodalService(
|
||||
chunkService interfaces.ChunkService,
|
||||
modelService interfaces.ModelService,
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ollamaService *ollama.OllamaService,
|
||||
) interfaces.TaskHandler {
|
||||
return &ImageMultimodalService{
|
||||
chunkService: chunkService,
|
||||
modelService: modelService,
|
||||
kbService: kbService,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ollamaService: ollamaService,
|
||||
}
|
||||
}
|
||||
|
||||
// Handle implements asynq handler for TypeImageMultimodal.
|
||||
func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) error {
|
||||
var payload types.ImageMultimodalPayload
|
||||
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
|
||||
return fmt.Errorf("unmarshal image multimodal payload: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ImageMultimodal] Processing image: chunk=%s, url=%s, local_path=%s, ocr=%v, caption=%v",
|
||||
payload.ChunkID, payload.ImageURL, payload.ImageLocalPath, payload.EnableOCR, payload.EnableCaption)
|
||||
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
|
||||
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve VLM: %w", err)
|
||||
}
|
||||
|
||||
imgBytes, err := os.ReadFile(payload.ImageLocalPath)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Local file %s not available (%v), trying to download from URL", payload.ImageLocalPath, err)
|
||||
imgBytes, err = downloadImageFromURL(payload.ImageURL)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[ImageMultimodal] Failed to download image from URL %s: %v", payload.ImageURL, err)
|
||||
return fmt.Errorf("read image: local file %s not found and URL download failed: %w", payload.ImageLocalPath, err)
|
||||
}
|
||||
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
|
||||
defer os.Remove(payload.ImageLocalPath)
|
||||
}
|
||||
|
||||
imageInfo := types.ImageInfo{
|
||||
URL: payload.ImageURL,
|
||||
OriginalURL: payload.ImageURL,
|
||||
}
|
||||
|
||||
if payload.EnableOCR {
|
||||
ocrText, ocrErr := vlmModel.Predict(ctx, imgBytes, vlmOCRPrompt)
|
||||
if ocrErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
|
||||
} else {
|
||||
ocrText = sanitizeOCRText(ocrText)
|
||||
if ocrText != "" {
|
||||
imageInfo.OCRText = ocrText
|
||||
} else {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if payload.EnableCaption {
|
||||
caption, capErr := vlmModel.Predict(ctx, imgBytes, vlmCaptionPrompt)
|
||||
if capErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
|
||||
} else if caption != "" {
|
||||
imageInfo.Caption = caption
|
||||
}
|
||||
}
|
||||
|
||||
// Build child chunks for OCR and caption results
|
||||
imageInfoJSON, _ := json.Marshal([]types.ImageInfo{imageInfo})
|
||||
var newChunks []*types.Chunk
|
||||
|
||||
if imageInfo.OCRText != "" {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: imageInfo.OCRText,
|
||||
ChunkType: types.ChunkTypeImageOCR,
|
||||
ParentChunkID: payload.ChunkID,
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
ImageInfo: string(imageInfoJSON),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
if imageInfo.Caption != "" {
|
||||
newChunks = append(newChunks, &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Content: imageInfo.Caption,
|
||||
ChunkType: types.ChunkTypeImageCaption,
|
||||
ParentChunkID: payload.ChunkID,
|
||||
IsEnabled: true,
|
||||
Flags: types.ChunkFlagRecommended,
|
||||
ImageInfo: string(imageInfoJSON),
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
if len(newChunks) == 0 {
|
||||
// Even if OCR/caption both failed, mark knowledge as completed
|
||||
s.finalizeImageKnowledge(ctx, payload, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Persist chunks
|
||||
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
|
||||
return fmt.Errorf("create multimodal chunks: %w", err)
|
||||
}
|
||||
for _, c := range newChunks {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
|
||||
c.ChunkType, c.ID, payload.ImageURL, len(c.Content))
|
||||
}
|
||||
|
||||
// Index chunks so they can be retrieved
|
||||
s.indexChunks(ctx, payload, newChunks)
|
||||
|
||||
// Update the parent text chunk's ImageInfo (mirrors old docreader behaviour)
|
||||
s.updateParentChunkImageInfo(ctx, payload, imageInfo)
|
||||
|
||||
// For standalone image files, use caption as the knowledge description
|
||||
// and mark the knowledge as completed (it was kept in "processing" until now).
|
||||
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// finalizeImageKnowledge updates the knowledge after multimodal processing:
|
||||
// - For standalone image files: sets Description from caption and marks ParseStatus as completed
|
||||
// (processChunks kept it in "processing" to wait for multimodal results).
|
||||
// - For images extracted from PDFs: no-op (description comes from summary generation).
|
||||
func (s *ImageMultimodalService) finalizeImageKnowledge(ctx context.Context, payload types.ImageMultimodalPayload, caption string) {
|
||||
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
|
||||
return
|
||||
}
|
||||
if knowledge == nil {
|
||||
return
|
||||
}
|
||||
if !IsImageType(knowledge.FileType) {
|
||||
return
|
||||
}
|
||||
|
||||
if caption != "" {
|
||||
knowledge.Description = caption
|
||||
}
|
||||
knowledge.ParseStatus = types.ParseStatusCompleted
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to finalize knowledge: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Finalized image knowledge %s (status=completed, description=%d chars)",
|
||||
payload.KnowledgeID, len(knowledge.Description))
|
||||
}
|
||||
}
|
||||
|
||||
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
|
||||
// so they can participate in semantic search.
|
||||
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil || kb == nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get KB for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get embedding model for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
indexInfoList := make([]*types.IndexInfo, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
||||
Content: chunk.Content,
|
||||
SourceID: chunk.ID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
})
|
||||
}
|
||||
|
||||
if err := engine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
|
||||
logger.Errorf(ctx, "[ImageMultimodal] Failed to index multimodal chunks: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Mark chunks as indexed.
|
||||
// Must re-fetch from DB because the in-memory objects lack auto-generated fields
|
||||
// (e.g. seq_id), and GORM Save would overwrite them with zero values.
|
||||
for _, chunk := range chunks {
|
||||
dbChunk, err := s.chunkService.GetChunkByIDOnly(ctx, chunk.ID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to fetch chunk %s for status update: %v", chunk.ID, err)
|
||||
continue
|
||||
}
|
||||
dbChunk.Status = int(types.ChunkStatusIndexed)
|
||||
if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
|
||||
}
|
||||
|
||||
// updateParentChunkImageInfo updates the parent text chunk's ImageInfo field,
|
||||
// replicating the behaviour of the old docreader flow where the parent chunk
|
||||
// carried the full image metadata (URL, OCR, caption).
|
||||
func (s *ImageMultimodalService) updateParentChunkImageInfo(ctx context.Context, payload types.ImageMultimodalPayload, imageInfo types.ImageInfo) {
|
||||
if payload.ChunkID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
|
||||
return
|
||||
}
|
||||
|
||||
var existingInfos []types.ImageInfo
|
||||
if chunk.ImageInfo != "" {
|
||||
_ = json.Unmarshal([]byte(chunk.ImageInfo), &existingInfos)
|
||||
}
|
||||
|
||||
found := false
|
||||
for i, info := range existingInfos {
|
||||
if info.URL == imageInfo.URL {
|
||||
existingInfos[i] = imageInfo
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
existingInfos = append(existingInfos, imageInfo)
|
||||
}
|
||||
|
||||
imageInfoJSON, _ := json.Marshal(existingInfos)
|
||||
chunk.ImageInfo = string(imageInfoJSON)
|
||||
chunk.UpdatedAt = time.Now()
|
||||
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to update parent chunk %s ImageInfo: %v", chunk.ID, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Updated parent chunk %s ImageInfo for image %s", chunk.ID, payload.ImageURL)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
|
||||
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
|
||||
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get knowledge base %s: %w", kbID, err)
|
||||
}
|
||||
if kb == nil {
|
||||
return nil, fmt.Errorf("knowledge base %s not found", kbID)
|
||||
}
|
||||
|
||||
vlmCfg := kb.VLMConfig
|
||||
if !vlmCfg.IsEnabled() {
|
||||
return nil, fmt.Errorf("VLM is not enabled for knowledge base %s", kbID)
|
||||
}
|
||||
|
||||
// New-style: resolve model through ModelService
|
||||
if vlmCfg.ModelID != "" {
|
||||
return s.modelService.GetVLMModel(ctx, vlmCfg.ModelID)
|
||||
}
|
||||
|
||||
// Legacy: create VLM from inline config
|
||||
return vlm.NewVLMFromLegacyConfig(vlmCfg, s.ollamaService)
|
||||
}
|
||||
|
||||
// downloadImageFromURL downloads image bytes from an HTTP(S) URL.
|
||||
func downloadImageFromURL(imageURL string) ([]byte, error) {
|
||||
if !strings.HasPrefix(imageURL, "http://") && !strings.HasPrefix(imageURL, "https://") {
|
||||
return nil, fmt.Errorf("unsupported URL scheme: %s", imageURL)
|
||||
}
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Get(imageURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("HTTP GET: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, imageURL)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
|
@ -32,7 +32,7 @@ type knowledgeBaseService struct {
|
|||
tenantRepo interfaces.TenantRepository
|
||||
fileSvc interfaces.FileService
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
asynqClient *asynq.Client
|
||||
asynqClient interfaces.TaskEnqueuer
|
||||
}
|
||||
|
||||
// NewKnowledgeBaseService creates a new knowledge base service
|
||||
|
|
@ -46,7 +46,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
|||
tenantRepo interfaces.TenantRepository,
|
||||
fileSvc interfaces.FileService,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
asynqClient *asynq.Client,
|
||||
asynqClient interfaces.TaskEnqueuer,
|
||||
) interfaces.KnowledgeBaseService {
|
||||
return &knowledgeBaseService{
|
||||
repo: repo,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
|
@ -405,5 +406,56 @@ func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.C
|
|||
return chatModel, nil
|
||||
}
|
||||
|
||||
// GetVLMModel retrieves and initializes a vision language model instance.
|
||||
func (s *modelService) GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error) {
|
||||
if modelId == "" {
|
||||
return nil, errors.New("model ID cannot be empty")
|
||||
}
|
||||
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
model, err := s.repo.GetByID(ctx, tenantID, modelId)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": modelId,
|
||||
"tenant_id": tenantID,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if model == nil {
|
||||
return nil, ErrModelNotFound
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Getting VLM model: %s, source: %s", model.Name, model.Source)
|
||||
|
||||
ifType := model.Parameters.InterfaceType
|
||||
if ifType == "" {
|
||||
if model.Source == types.ModelSourceLocal {
|
||||
ifType = "ollama"
|
||||
} else {
|
||||
ifType = "openai"
|
||||
}
|
||||
}
|
||||
|
||||
vlmModel, err := vlm.NewVLM(&vlm.Config{
|
||||
ModelID: model.ID,
|
||||
APIKey: model.Parameters.APIKey,
|
||||
BaseURL: model.Parameters.BaseURL,
|
||||
ModelName: model.Name,
|
||||
Source: model.Source,
|
||||
InterfaceType: ifType,
|
||||
}, s.ollamaService)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": model.ID,
|
||||
"model_name": model.Name,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return vlmModel, nil
|
||||
}
|
||||
|
||||
// Note: default model selection logic has been removed; models no longer
|
||||
// maintain a per-type default flag at the service layer.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
htmlTagPattern = regexp.MustCompile(`<[^>]+>`)
|
||||
codeBlockPattern = regexp.MustCompile("(?s)^\\s*```[a-zA-Z]*\\s*\n(.*?)\n\\s*```\\s*$")
|
||||
htmlDocPattern = regexp.MustCompile(`(?i)^\s*(<\!DOCTYPE|<html|<body|<div|<p[\s>]|<table|<h[1-6][\s>])`)
|
||||
multipleNewlines = regexp.MustCompile(`\n{3,}`)
|
||||
knownEmptyReplies = []string{
|
||||
"无文字内容",
|
||||
"无法识别",
|
||||
"no text",
|
||||
"no content",
|
||||
"empty",
|
||||
"图片中没有文字",
|
||||
"图片中没有可识别的文字",
|
||||
}
|
||||
)
|
||||
|
||||
// sanitizeOCRText cleans up VLM OCR output by stripping HTML wrappers,
|
||||
// converting HTML to markdown, and filtering out useless responses.
|
||||
func sanitizeOCRText(raw string) string {
|
||||
text := strings.TrimSpace(raw)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
text = stripMarkdownCodeBlock(text)
|
||||
|
||||
// If stripping HTML tags leaves almost no text, the response is useless
|
||||
// (e.g. "<html><body><div class="image"><img/></div></body></html>").
|
||||
plainText := strings.TrimSpace(htmlTagPattern.ReplaceAllString(text, ""))
|
||||
if len(plainText) < 10 && htmlTagPattern.MatchString(text) {
|
||||
return ""
|
||||
}
|
||||
|
||||
if looksLikeHTML(text) {
|
||||
text = ocrHTMLToMarkdown(text)
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
if isKnownEmptyReply(text) {
|
||||
return ""
|
||||
}
|
||||
|
||||
text = multipleNewlines.ReplaceAllString(text, "\n\n")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// stripMarkdownCodeBlock removes a markdown code-fence wrapper that some
|
||||
// models add around their output (e.g. ```html\n...\n``` or ```markdown\n...\n```).
|
||||
func stripMarkdownCodeBlock(text string) string {
|
||||
if m := codeBlockPattern.FindStringSubmatch(text); len(m) == 2 {
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// looksLikeHTML returns true when the text appears to be an HTML document
|
||||
// or contains a significant amount of HTML tags.
|
||||
func looksLikeHTML(text string) bool {
|
||||
if htmlDocPattern.MatchString(text) {
|
||||
return true
|
||||
}
|
||||
tags := htmlTagPattern.FindAllString(text, -1)
|
||||
if len(tags) == 0 {
|
||||
return false
|
||||
}
|
||||
tagChars := 0
|
||||
for _, t := range tags {
|
||||
tagChars += len(t)
|
||||
}
|
||||
return float64(tagChars)/float64(len(text)) > 0.3
|
||||
}
|
||||
|
||||
// ocrHTMLToMarkdown converts HTML content to markdown, falling back to the
|
||||
// original text on failure.
|
||||
func ocrHTMLToMarkdown(content string) string {
|
||||
md, err := htmltomd.ConvertString(content)
|
||||
if err != nil {
|
||||
return content
|
||||
}
|
||||
return md
|
||||
}
|
||||
|
||||
// isKnownEmptyReply checks whether the text matches a known "no content"
|
||||
// reply pattern that VLM models produce when the image has no text.
|
||||
func isKnownEmptyReply(text string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(text))
|
||||
for _, phrase := range knownEmptyReplies {
|
||||
if lower == strings.ToLower(phrase) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
package service
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSanitizeOCRText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "whitespace only",
|
||||
input: " \n\t ",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "pure HTML skeleton with no text",
|
||||
input: `<html><body><div class="image"><img/></div></body></html>`,
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "HTML with only whitespace text",
|
||||
input: "<html><body> \n </body></html>",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "valid markdown passes through",
|
||||
input: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
|
||||
want: "# 标题\n\n这是一段正文,包含一些内容。\n\n| 列1 | 列2 |\n| --- | --- |\n| 数据1 | 数据2 |",
|
||||
},
|
||||
{
|
||||
name: "code block wrapper stripped",
|
||||
input: "```markdown\n# 文档标题\n\n正文内容在这里。\n```",
|
||||
want: "# 文档标题\n\n正文内容在这里。",
|
||||
},
|
||||
{
|
||||
name: "html code block wrapper stripped",
|
||||
input: "```html\n<p>这是一段内容</p>\n```",
|
||||
want: "这是一段内容",
|
||||
},
|
||||
{
|
||||
name: "HTML document converted to markdown",
|
||||
input: "<html><body><h1>标题</h1><p>这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。</p></body></html>",
|
||||
want: "# 标题\n\n这是一段很长的正文内容,用来测试 HTML 到 Markdown 的转换。",
|
||||
},
|
||||
{
|
||||
name: "known empty reply - Chinese",
|
||||
input: "无文字内容",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "known empty reply - no text",
|
||||
input: "No text",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "known empty reply - 图片中没有文字",
|
||||
input: "图片中没有文字",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "plain text with minimal HTML not converted",
|
||||
input: "这是一段正常文本,价格 <100 元。",
|
||||
want: "这是一段正常文本,价格 <100 元。",
|
||||
},
|
||||
{
|
||||
name: "multiple blank lines collapsed",
|
||||
input: "段落一\n\n\n\n\n段落二",
|
||||
want: "段落一\n\n段落二",
|
||||
},
|
||||
{
|
||||
name: "HTML with substantial text content is converted",
|
||||
input: "<div><h2>报告摘要</h2><p>本季度营收同比增长 15%,净利润达到 2.3 亿元。</p><table><tr><th>指标</th><th>数值</th></tr><tr><td>营收</td><td>10亿</td></tr></table></div>",
|
||||
want: "", // placeholder; will be checked for non-empty
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := sanitizeOCRText(tt.input)
|
||||
|
||||
if tt.name == "HTML with substantial text content is converted" {
|
||||
if got == "" {
|
||||
t.Errorf("sanitizeOCRText() returned empty for substantial HTML content")
|
||||
}
|
||||
if got == tt.input {
|
||||
t.Errorf("sanitizeOCRText() did not convert HTML, got original")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if got != tt.want {
|
||||
t.Errorf("sanitizeOCRText() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripMarkdownCodeBlock(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "no code block",
|
||||
input: "just normal text",
|
||||
want: "just normal text",
|
||||
},
|
||||
{
|
||||
name: "markdown code block",
|
||||
input: "```markdown\n# Title\nContent here\n```",
|
||||
want: "# Title\nContent here",
|
||||
},
|
||||
{
|
||||
name: "html code block",
|
||||
input: "```html\n<p>hello</p>\n```",
|
||||
want: "<p>hello</p>",
|
||||
},
|
||||
{
|
||||
name: "plain code block",
|
||||
input: "```\nsome text\n```",
|
||||
want: "some text",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := stripMarkdownCodeBlock(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("stripMarkdownCodeBlock() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLooksLikeHTML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "HTML document",
|
||||
input: "<html><body><p>text</p></body></html>",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "DOCTYPE",
|
||||
input: "<!DOCTYPE html><html><body></body></html>",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "body tag",
|
||||
input: "<body><p>content</p></body>",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "plain markdown",
|
||||
input: "# Title\n\nSome paragraph text",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "text with minor HTML",
|
||||
input: "This is mostly text with a <b>bold</b> word.",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "heavy HTML tags",
|
||||
input: "<div><p><span>x</span></p></div><div><p><span>y</span></p></div>",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := looksLikeHTML(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("looksLikeHTML() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsKnownEmptyReply(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{"无文字内容", true},
|
||||
{"无法识别", true},
|
||||
{"no text", true},
|
||||
{"No Text", true},
|
||||
{"NO CONTENT", true},
|
||||
{"empty", true},
|
||||
{"这是正常内容", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := isKnownEmptyReply(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("isKnownEmptyReply(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -25,7 +25,7 @@ type knowledgeTagService struct {
|
|||
chunkRepo interfaces.ChunkRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
modelService interfaces.ModelService
|
||||
task *asynq.Client
|
||||
task interfaces.TaskEnqueuer
|
||||
kbShareService interfaces.KBShareService
|
||||
}
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ func NewKnowledgeTagService(
|
|||
chunkRepo interfaces.ChunkRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
modelService interfaces.ModelService,
|
||||
task *asynq.Client,
|
||||
task interfaces.TaskEnqueuer,
|
||||
kbShareService interfaces.KBShareService,
|
||||
) (interfaces.KnowledgeTagService, error) {
|
||||
return &knowledgeTagService{
|
||||
|
|
|
|||
|
|
@ -29,8 +29,12 @@ type Config struct {
|
|||
PromptTemplates *PromptTemplatesConfig `yaml:"prompt_templates" json:"prompt_templates"`
|
||||
}
|
||||
|
||||
// DocReaderConfig configures the document parser client (gRPC or HTTP).
|
||||
type DocReaderConfig struct {
|
||||
// Addr: for gRPC it is the server address (e.g. "localhost:50051"); for HTTP it is the base URL (e.g. "http://localhost:8080").
|
||||
Addr string `yaml:"addr" json:"addr"`
|
||||
// Transport: "grpc" (default) or "http"
|
||||
Transport string `yaml:"transport" json:"transport"`
|
||||
}
|
||||
|
||||
type VectorDatabaseConfig struct {
|
||||
|
|
|
|||
|
|
@ -9,11 +9,13 @@ import (
|
|||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
_ "github.com/duckdb/duckdb-go/v2"
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
|
|
@ -25,9 +27,9 @@ import (
|
|||
"go.uber.org/dig"
|
||||
"google.golang.org/grpc"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/Tencent/WeKnora/docreader/client"
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
|
|
@ -36,6 +38,7 @@ import (
|
|||
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
||||
sqliteRetrieverRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/sqlite"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
|
|
@ -48,6 +51,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/handler"
|
||||
"github.com/Tencent/WeKnora/internal/handler/session"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/mcp"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
|
|
@ -94,6 +98,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
// External service clients
|
||||
logger.Debugf(ctx, "[Container] Registering external service clients...")
|
||||
must(container.Provide(initDocReaderClient))
|
||||
must(container.Provide(initImageResolver))
|
||||
must(container.Provide(initOllamaService))
|
||||
must(container.Provide(initNeo4jClient))
|
||||
must(container.Provide(stream.NewStreamManager))
|
||||
|
|
@ -146,6 +151,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
// Extract services - register individual extracters with names
|
||||
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
|
||||
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
|
||||
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
|
||||
|
||||
must(container.Provide(service.NewMessageService))
|
||||
must(container.Provide(service.NewMCPServiceService))
|
||||
|
|
@ -169,9 +175,16 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
logger.Debugf(ctx, "[Container] Registering session service...")
|
||||
must(container.Provide(service.NewSessionService))
|
||||
|
||||
logger.Debugf(ctx, "[Container] Registering asynq client and server...")
|
||||
must(container.Provide(router.NewAsyncqClient))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
logger.Debugf(ctx, "[Container] Registering task enqueuer...")
|
||||
redisAvailable := os.Getenv("REDIS_ADDR") != ""
|
||||
if redisAvailable {
|
||||
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
} else {
|
||||
syncExec := router.NewSyncTaskExecutor()
|
||||
must(container.Provide(func() interfaces.TaskEnqueuer { return syncExec }))
|
||||
must(container.Provide(func() *router.SyncTaskExecutor { return syncExec }))
|
||||
}
|
||||
|
||||
// Chat pipeline components for processing chat requests
|
||||
logger.Debugf(ctx, "[Container] Registering chat pipeline plugins...")
|
||||
|
|
@ -218,9 +231,13 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
logger.Debugf(ctx, "[Container] HTTP handlers registered")
|
||||
|
||||
// Router configuration
|
||||
logger.Debugf(ctx, "[Container] Registering router and starting asynq server...")
|
||||
logger.Debugf(ctx, "[Container] Registering router and starting task server...")
|
||||
must(container.Provide(router.NewRouter))
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
if redisAvailable {
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
} else {
|
||||
must(container.Invoke(router.RegisterSyncHandlers))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Container] Container initialization completed successfully")
|
||||
return container
|
||||
|
|
@ -249,19 +266,23 @@ func initTracer() (*tracing.Tracer, error) {
|
|||
}
|
||||
|
||||
func initRedisClient() (*redis.Client, error) {
|
||||
redisAddr := os.Getenv("REDIS_ADDR")
|
||||
if redisAddr == "" {
|
||||
logger.Infof(context.Background(), "[Redis] No REDIS_ADDR configured, Redis disabled (Lite mode)")
|
||||
return nil, nil
|
||||
}
|
||||
db, err := strconv.Atoi(os.Getenv("REDIS_DB"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
db = 0
|
||||
}
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: os.Getenv("REDIS_ADDR"),
|
||||
Addr: redisAddr,
|
||||
Username: os.Getenv("REDIS_USERNAME"),
|
||||
Password: os.Getenv("REDIS_PASSWORD"),
|
||||
DB: db,
|
||||
})
|
||||
|
||||
// 验证连接
|
||||
_, err = client.Ping(context.Background()).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("连接Redis失败: %w", err)
|
||||
|
|
@ -271,6 +292,10 @@ func initRedisClient() (*redis.Client, error) {
|
|||
}
|
||||
|
||||
func initContextStorage(redisClient *redis.Client) (llmcontext.ContextStorage, error) {
|
||||
if redisClient == nil {
|
||||
logger.Infof(context.Background(), "[ContextStorage] Redis not available, using in-memory storage")
|
||||
return llmcontext.NewMemoryStorage(), nil
|
||||
}
|
||||
storage, err := llmcontext.NewRedisStorage(redisClient, 24*time.Hour, "context:")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -334,6 +359,21 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
|||
os.Getenv("DB_PORT"),
|
||||
os.Getenv("DB_NAME"),
|
||||
)
|
||||
case "sqlite":
|
||||
dbPath := os.Getenv("DB_PATH")
|
||||
if dbPath == "" {
|
||||
dbPath = "./data/weknora.db"
|
||||
}
|
||||
if dir := filepath.Dir(dbPath); dir != "." && dir != "" {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create SQLite data directory %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
sqlite_vec.Auto()
|
||||
dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on"
|
||||
dialector = sqlite.Open(dsn)
|
||||
migrateDSN = "sqlite3://" + dbPath
|
||||
logger.Infof(context.Background(), "DB Config: driver=sqlite path=%s", dbPath)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER"))
|
||||
}
|
||||
|
|
@ -342,6 +382,16 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if os.Getenv("DB_DRIVER") == "sqlite" {
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get underlying sql.DB: %w", err)
|
||||
}
|
||||
if err := sqlDB.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("failed to ping SQLite database: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Run database migrations automatically (optional, can be disabled via env var)
|
||||
// To disable auto-migration, set AUTO_MIGRATE=false
|
||||
// To enable auto-recovery from dirty state, set AUTO_RECOVER_DIRTY=true
|
||||
|
|
@ -374,7 +424,14 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
|||
}
|
||||
|
||||
// Configure connection pool parameters
|
||||
sqlDB.SetMaxIdleConns(10)
|
||||
if os.Getenv("DB_DRIVER") == "sqlite" {
|
||||
// SQLite only supports one concurrent writer even in WAL mode.
|
||||
// Limiting to a single open connection serialises all DB access and
|
||||
// prevents "database is locked" errors from concurrent goroutines.
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
} else {
|
||||
sqlDB.SetMaxIdleConns(10)
|
||||
}
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute)
|
||||
|
||||
return db, nil
|
||||
|
|
@ -390,7 +447,11 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
|||
// - Configured file service implementation
|
||||
// - Error if initialization fails
|
||||
func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
switch os.Getenv("STORAGE_TYPE") {
|
||||
storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE"))
|
||||
if storageType == "" {
|
||||
storageType = "local"
|
||||
}
|
||||
switch storageType {
|
||||
case "minio":
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" ||
|
||||
os.Getenv("MINIO_ACCESS_KEY_ID") == "" ||
|
||||
|
|
@ -419,8 +480,8 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
|||
os.Getenv("COS_SECRET_ID"),
|
||||
os.Getenv("COS_SECRET_KEY"),
|
||||
os.Getenv("COS_PATH_PREFIX"),
|
||||
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
|
||||
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
os.Getenv("COS_TEMP_BUCKET_NAME"),
|
||||
os.Getenv("COS_TEMP_REGION"),
|
||||
)
|
||||
case "tos":
|
||||
if os.Getenv("TOS_ENDPOINT") == "" ||
|
||||
|
|
@ -441,11 +502,15 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
|||
os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
)
|
||||
case "local":
|
||||
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
}
|
||||
return file.NewLocalFileService(baseDir), nil
|
||||
case "dummy":
|
||||
return file.NewDummyFileService(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage type: %s", os.Getenv("STORAGE_TYPE"))
|
||||
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -474,6 +539,16 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
|||
log.Infof("Register postgres retrieve engine success")
|
||||
}
|
||||
}
|
||||
if slices.Contains(retrieveDriver, "sqlite") {
|
||||
sqliteRepo := sqliteRetrieverRepo.NewSQLiteRetrieveEngineRepository(db)
|
||||
if err := registry.Register(
|
||||
retriever.NewKVHybridRetrieveEngine(sqliteRepo, types.SQLiteRetrieverEngineType),
|
||||
); err != nil {
|
||||
log.Errorf("Register sqlite retrieve engine failed: %v", err)
|
||||
} else {
|
||||
log.Infof("Register sqlite retrieve engine success")
|
||||
}
|
||||
}
|
||||
if slices.Contains(retrieveDriver, "elasticsearch_v8") {
|
||||
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
|
||||
Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")},
|
||||
|
|
@ -639,21 +714,39 @@ func registerPoolCleanup(pool *ants.Pool, cleaner interfaces.ResourceCleaner) {
|
|||
})
|
||||
}
|
||||
|
||||
// initDocReaderClient initializes the document reader client
|
||||
// Creates a client for interacting with the document reader service
|
||||
// Parameters:
|
||||
// - cfg: Application configuration
|
||||
//
|
||||
// Returns:
|
||||
// - Configured document reader client
|
||||
// - Error if initialization fails
|
||||
func initDocReaderClient(cfg *config.Config) (*client.Client, error) {
|
||||
// Use the DocReader URL from environment or config
|
||||
docReaderURL := os.Getenv("DOCREADER_ADDR")
|
||||
if docReaderURL == "" && cfg.DocReader != nil {
|
||||
docReaderURL = cfg.DocReader.Addr
|
||||
// initDocReaderClient initializes the DocumentReader client (lightweight API).
|
||||
func initDocReaderClient(cfg *config.Config) (interfaces.DocumentReader, error) {
|
||||
addr := strings.TrimSpace(os.Getenv("DOCREADER_ADDR"))
|
||||
transport := strings.TrimSpace(os.Getenv("DOCREADER_TRANSPORT"))
|
||||
if transport == "" {
|
||||
transport = "grpc"
|
||||
}
|
||||
return client.NewClient(docReaderURL)
|
||||
if addr == "" {
|
||||
logger.Infof(context.Background(), "[DocConverter] No DOCREADER_ADDR configured, starting disconnected")
|
||||
}
|
||||
transport = strings.ToLower(transport)
|
||||
switch transport {
|
||||
case "http", "https":
|
||||
if addr != "" && !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") {
|
||||
addr = "http://" + addr
|
||||
}
|
||||
return docparser.NewHTTPDocumentReader(addr)
|
||||
default:
|
||||
return docparser.NewGRPCDocumentReader(addr)
|
||||
}
|
||||
}
|
||||
|
||||
// initImageResolver creates the ImageResolver for storing images from DocReader.
|
||||
func initImageResolver(cfg *config.Config) *docparser.ImageResolver {
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
}
|
||||
urlPrefix := os.Getenv("LOCAL_STORAGE_URL_PREFIX")
|
||||
if urlPrefix == "" {
|
||||
urlPrefix = "/files"
|
||||
}
|
||||
return docparser.NewImageResolver(baseDir, urlPrefix)
|
||||
}
|
||||
|
||||
// initOllamaService initializes the Ollama service client
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
_ "github.com/golang-migrate/migrate/v4/database/postgres"
|
||||
_ "github.com/golang-migrate/migrate/v4/database/sqlite3"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
)
|
||||
|
||||
|
|
@ -30,8 +32,10 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error {
|
|||
|
||||
logger.Infof(ctx, "Starting database migration...")
|
||||
|
||||
// Use versioned migrations directory
|
||||
migrationsPath := "file://migrations/versioned"
|
||||
if strings.HasPrefix(dsn, "sqlite3://") {
|
||||
migrationsPath = "file://migrations/sqlite"
|
||||
}
|
||||
|
||||
m, err := migrate.New(migrationsPath, dsn)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/docreader/client"
|
||||
"github.com/Tencent/WeKnora/docreader/proto"
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
|
|
@ -57,7 +55,7 @@ type InitializationHandler struct {
|
|||
kbRepository interfaces.KnowledgeBaseRepository
|
||||
knowledgeService interfaces.KnowledgeService
|
||||
ollamaService *ollama.OllamaService
|
||||
docReaderClient *client.Client
|
||||
documentReader interfaces.DocumentReader
|
||||
pooler embedding.EmbedderPooler
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +68,7 @@ func NewInitializationHandler(
|
|||
kbRepository interfaces.KnowledgeBaseRepository,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
ollamaService *ollama.OllamaService,
|
||||
docReaderClient *client.Client,
|
||||
documentReader interfaces.DocumentReader,
|
||||
pooler embedding.EmbedderPooler,
|
||||
) *InitializationHandler {
|
||||
return &InitializationHandler{
|
||||
|
|
@ -81,7 +79,7 @@ func NewInitializationHandler(
|
|||
kbRepository: kbRepository,
|
||||
knowledgeService: knowledgeService,
|
||||
ollamaService: ollamaService,
|
||||
docReaderClient: docReaderClient,
|
||||
documentReader: documentReader,
|
||||
pooler: pooler,
|
||||
}
|
||||
}
|
||||
|
|
@ -94,30 +92,20 @@ type KBModelConfigRequest struct {
|
|||
|
||||
// 文档分块配置
|
||||
DocumentSplitting struct {
|
||||
ChunkSize int `json:"chunkSize"`
|
||||
ChunkOverlap int `json:"chunkOverlap"`
|
||||
Separators []string `json:"separators"`
|
||||
ChunkSize int `json:"chunkSize"`
|
||||
ChunkOverlap int `json:"chunkOverlap"`
|
||||
Separators []string `json:"separators"`
|
||||
ParserEngineRules []types.ParserEngineRule `json:"parserEngineRules,omitempty"`
|
||||
} `json:"documentSplitting"`
|
||||
|
||||
// 多模态配置
|
||||
// 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置)
|
||||
Multimodal struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
StorageType string `json:"storageType"` // "cos" or "minio"
|
||||
COS *struct {
|
||||
SecretID string `json:"secretId"`
|
||||
SecretKey string `json:"secretKey"`
|
||||
Region string `json:"region"`
|
||||
BucketName string `json:"bucketName"`
|
||||
AppID string `json:"appId"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
} `json:"cos"`
|
||||
Minio *struct {
|
||||
BucketName string `json:"bucketName"`
|
||||
UseSSL bool `json:"useSSL"`
|
||||
PathPrefix string `json:"pathPrefix"`
|
||||
} `json:"minio"`
|
||||
Enabled bool `json:"enabled"`
|
||||
} `json:"multimodal"`
|
||||
|
||||
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
|
||||
StorageProvider string `json:"storageProvider"`
|
||||
|
||||
// 知识图谱配置
|
||||
NodeExtract struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
|
|
@ -300,38 +288,22 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
|||
if len(req.DocumentSplitting.Separators) > 0 {
|
||||
kb.ChunkingConfig.Separators = req.DocumentSplitting.Separators
|
||||
}
|
||||
kb.ChunkingConfig.ParserEngineRules = req.DocumentSplitting.ParserEngineRules
|
||||
|
||||
// 更新多模态配置
|
||||
if req.Multimodal.Enabled {
|
||||
switch strings.ToLower(req.Multimodal.StorageType) {
|
||||
case "cos":
|
||||
if req.Multimodal.COS != nil {
|
||||
kb.StorageConfig = types.StorageConfig{
|
||||
SecretID: req.Multimodal.COS.SecretID,
|
||||
SecretKey: req.Multimodal.COS.SecretKey,
|
||||
Region: req.Multimodal.COS.Region,
|
||||
BucketName: req.Multimodal.COS.BucketName,
|
||||
AppID: req.Multimodal.COS.AppID,
|
||||
PathPrefix: req.Multimodal.COS.PathPrefix,
|
||||
Provider: "cos",
|
||||
}
|
||||
}
|
||||
case "minio":
|
||||
if req.Multimodal.Minio != nil {
|
||||
kb.StorageConfig = types.StorageConfig{
|
||||
BucketName: req.Multimodal.Minio.BucketName,
|
||||
PathPrefix: req.Multimodal.Minio.PathPrefix,
|
||||
Provider: "minio",
|
||||
SecretID: os.Getenv("MINIO_ACCESS_KEY_ID"),
|
||||
SecretKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"),
|
||||
}
|
||||
}
|
||||
}
|
||||
// VLM model already set above
|
||||
} else {
|
||||
// 多模态未启用时,清空存储配置
|
||||
kb.StorageConfig = types.StorageConfig{}
|
||||
kb.VLMConfig.ModelID = ""
|
||||
}
|
||||
|
||||
// 存储引擎:仅写入 provider,参数从租户全局 StorageEngineConfig 读取
|
||||
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
|
||||
if provider == "" {
|
||||
provider = "local"
|
||||
}
|
||||
kb.StorageConfig = types.StorageConfig{Provider: provider}
|
||||
|
||||
// 更新知识图谱配置
|
||||
if req.NodeExtract.Enabled {
|
||||
// 转换 Nodes 和 Relations 为指针类型
|
||||
|
|
@ -1910,97 +1882,43 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// testMultimodalWithDocReader 调用docreader服务进行多模态处理
|
||||
// testMultimodalWithDocReader uses DocumentReader.Read for document reading,
|
||||
// then returns basic information about the result.
|
||||
func (h *InitializationHandler) testMultimodalWithDocReader(
|
||||
ctx context.Context,
|
||||
imageContent []byte, filename string,
|
||||
chunkSize, chunkOverlap int32, separators []string,
|
||||
req *testMultimodalForm,
|
||||
) (map[string]string, error) {
|
||||
// 获取文件扩展名
|
||||
fileExt := ""
|
||||
if idx := strings.LastIndex(filename, "."); idx != -1 {
|
||||
fileExt = strings.ToLower(filename[idx+1:])
|
||||
}
|
||||
|
||||
// 检查docreader服务配置
|
||||
if h.docReaderClient == nil {
|
||||
if h.documentReader == nil {
|
||||
return nil, fmt.Errorf("DocReader service not configured")
|
||||
}
|
||||
|
||||
// 构造请求
|
||||
request := &proto.ReadFromFileRequest{
|
||||
requestID, _ := ctx.Value(types.RequestIDContextKey).(string)
|
||||
|
||||
readResult, err := h.documentReader.Read(ctx, &types.ReadRequest{
|
||||
FileContent: imageContent,
|
||||
FileName: filename,
|
||||
FileType: fileExt,
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: chunkSize,
|
||||
ChunkOverlap: chunkOverlap,
|
||||
Separators: separators,
|
||||
EnableMultimodal: true, // 启用多模态处理
|
||||
VlmConfig: &proto.VLMConfig{
|
||||
ModelName: req.VLMModel,
|
||||
BaseUrl: req.VLMBaseURL,
|
||||
ApiKey: req.VLMAPIKey,
|
||||
InterfaceType: req.VLMInterfaceType,
|
||||
},
|
||||
},
|
||||
RequestId: ctx.Value(types.RequestIDContextKey).(string),
|
||||
}
|
||||
|
||||
// 设置对象存储配置(通用)
|
||||
switch strings.ToLower(req.StorageType) {
|
||||
case "cos":
|
||||
request.ReadConfig.StorageConfig = &proto.StorageConfig{
|
||||
Provider: proto.StorageProvider_COS,
|
||||
Region: req.COSRegion,
|
||||
BucketName: req.COSBucketName,
|
||||
AccessKeyId: req.COSSecretID,
|
||||
SecretAccessKey: req.COSSecretKey,
|
||||
AppId: req.COSAppID,
|
||||
PathPrefix: req.COSPathPrefix,
|
||||
}
|
||||
case "minio":
|
||||
request.ReadConfig.StorageConfig = &proto.StorageConfig{
|
||||
Provider: proto.StorageProvider_MINIO,
|
||||
BucketName: req.MinioBucketName,
|
||||
PathPrefix: req.MinioPathPrefix,
|
||||
AccessKeyId: os.Getenv("MINIO_ACCESS_KEY_ID"),
|
||||
SecretAccessKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"),
|
||||
}
|
||||
}
|
||||
|
||||
// 调用docreader服务
|
||||
response, err := h.docReaderClient.ReadFromFile(ctx, request)
|
||||
RequestID: requestID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("调用DocReader服务失败: %v", err)
|
||||
}
|
||||
|
||||
if response.Error != "" {
|
||||
return nil, fmt.Errorf("DocReader服务返回错误: %s", response.Error)
|
||||
if readResult.Error != "" {
|
||||
return nil, fmt.Errorf("DocReader服务返回错误: %s", readResult.Error)
|
||||
}
|
||||
|
||||
// 处理响应,提取Caption和OCR信息
|
||||
result := make(map[string]string)
|
||||
var allCaptions, allOCRTexts []string
|
||||
|
||||
for _, chunk := range response.Chunks {
|
||||
if len(chunk.Images) > 0 {
|
||||
for _, image := range chunk.Images {
|
||||
if image.Caption != "" {
|
||||
allCaptions = append(allCaptions, image.Caption)
|
||||
}
|
||||
if image.OcrText != "" {
|
||||
allOCRTexts = append(allOCRTexts, image.OcrText)
|
||||
}
|
||||
}
|
||||
}
|
||||
result := map[string]string{
|
||||
"markdown": readResult.MarkdownContent,
|
||||
"caption": "",
|
||||
"ocr": "",
|
||||
}
|
||||
|
||||
// 合并所有Caption和OCR结果
|
||||
result["caption"] = strings.Join(allCaptions, "; ")
|
||||
result["ocr"] = strings.Join(allOCRTexts, "; ")
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -738,6 +738,102 @@ func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) {
|
|||
})
|
||||
}
|
||||
|
||||
// mimeTypeByExt returns the MIME type for a given file extension.
|
||||
func mimeTypeByExt(filename string) string {
|
||||
ext := strings.ToLower(filename)
|
||||
if idx := strings.LastIndex(ext, "."); idx >= 0 {
|
||||
ext = ext[idx:]
|
||||
} else {
|
||||
ext = ""
|
||||
}
|
||||
m := map[string]string{
|
||||
".pdf": "application/pdf",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".doc": "application/msword",
|
||||
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
".ppt": "application/vnd.ms-powerpoint",
|
||||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".xls": "application/vnd.ms-excel",
|
||||
".csv": "text/csv",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".gif": "image/gif",
|
||||
".bmp": "image/bmp",
|
||||
".webp": "image/webp",
|
||||
".svg": "image/svg+xml",
|
||||
".tiff": "image/tiff",
|
||||
".txt": "text/plain; charset=utf-8",
|
||||
".md": "text/markdown; charset=utf-8",
|
||||
".markdown": "text/markdown; charset=utf-8",
|
||||
".json": "application/json; charset=utf-8",
|
||||
".xml": "application/xml; charset=utf-8",
|
||||
".html": "text/html; charset=utf-8",
|
||||
".css": "text/css; charset=utf-8",
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".ts": "text/typescript; charset=utf-8",
|
||||
".py": "text/x-python; charset=utf-8",
|
||||
".go": "text/x-go; charset=utf-8",
|
||||
".java": "text/x-java; charset=utf-8",
|
||||
".yaml": "text/yaml; charset=utf-8",
|
||||
".yml": "text/yaml; charset=utf-8",
|
||||
".sh": "text/x-shellscript; charset=utf-8",
|
||||
}
|
||||
if ct, ok := m[ext]; ok {
|
||||
return ct
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
// PreviewKnowledgeFile godoc
|
||||
// @Summary 预览知识文件
|
||||
// @Description 返回知识条目关联的原始文件,Content-Type 根据文件类型设置,用于浏览器内嵌预览
|
||||
// @Tags 知识管理
|
||||
// @Accept json
|
||||
// @Produce application/pdf,image/jpeg,image/png,text/plain
|
||||
// @Param id path string true "知识ID"
|
||||
// @Success 200 {file} file "文件内容"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Security Bearer
|
||||
// @Security ApiKeyAuth
|
||||
// @Router /knowledge/{id}/preview [get]
|
||||
func (h *KnowledgeHandler) PreviewKnowledgeFile(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
id := secutils.SanitizeForLog(c.Param("id"))
|
||||
if id == "" {
|
||||
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
|
||||
return
|
||||
}
|
||||
|
||||
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleViewer)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
file, filename, err := h.kgService.GetKnowledgeFile(effCtx, id)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error()))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
contentType := mimeTypeByExt(filename)
|
||||
c.Header("Content-Type", contentType)
|
||||
c.Header("Content-Disposition", fmt.Sprintf("inline; filename=%s", filename))
|
||||
c.Header("Cache-Control", "private, max-age=3600")
|
||||
|
||||
c.Stream(func(w io.Writer) bool {
|
||||
if _, err := io.Copy(w, file); err != nil {
|
||||
logger.Errorf(ctx, "Failed to stream preview: %v", err)
|
||||
return false
|
||||
}
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
// GetKnowledgeBatchRequest defines parameters for batch knowledge retrieval
|
||||
type GetKnowledgeBatchRequest struct {
|
||||
IDs []string `form:"ids" binding:"required"` // List of knowledge IDs
|
||||
|
|
|
|||