feat(knowledge): add batch file downloads (#3300)

* feat(knowledge): add batch file downloads

* fix(knowledge): harden batch file downloads

Skip entries without original files, keep folder paths in the ZIP, map
client cancel to 400, and cap concurrent archives so a full-select
download is usable without 500s or disk pile-up.

* fix(knowledge): format batch-download tests for pre-push lint

---------

Co-authored-by: wizardchen <wizardchen@tencent.com>
main
ttommybot 2026-09-16 15:40:10 +08:00 committed by GitHub
parent 079e96ae64
commit a7d3545297
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 1387 additions and 22 deletions

View File

@ -91,6 +91,12 @@ knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), kb.ID,
if err != nil {
// 处理错误
}
// 将同一知识库中的原始文件打包下载为 ZIP(最多 200 个 ID,合计 512 MiB)
err = apiClient.DownloadKnowledgeFiles(context.Background(), kb.ID, []string{knowledge.ID}, "knowledge-files.zip")
if err != nil {
// 处理错误
}
```
### 示例:创建会话并进行问答

View File

@ -91,6 +91,12 @@ knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), kb.ID,
if err != nil {
// Handle error
}
// Download original files from one knowledge base as a ZIP (max 200 IDs, 512 MiB)
err = apiClient.DownloadKnowledgeFiles(context.Background(), kb.ID, []string{knowledge.ID}, "knowledge-files.zip")
if err != nil {
// Handle error
}
```
### Example: Create Session and Chat

View File

@ -557,6 +557,52 @@ func (c *Client) OpenKnowledgeFile(ctx context.Context, knowledgeID string) (str
return filename, resp.Body, nil
}
// BatchDownloadKnowledgeRequest selects documents in one knowledge base for ZIP download.
type BatchDownloadKnowledgeRequest struct {
IDs []string `json:"ids"`
}
// DownloadKnowledgeFiles downloads a ZIP of original files for the given
// knowledge IDs to destPath. On any error after the file is opened, the
// partial file is removed.
func (c *Client) DownloadKnowledgeFiles(ctx context.Context, knowledgeBaseID string, ids []string, destPath string) error {
_, body, err := c.OpenKnowledgeFilesArchive(ctx, knowledgeBaseID, ids)
if err != nil {
return err
}
defer body.Close()
out, err := os.Create(destPath)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
if _, err := io.Copy(out, body); err != nil {
_ = out.Close()
_ = os.Remove(destPath)
return fmt.Errorf("failed to copy response body: %w", err)
}
return out.Close()
}
// OpenKnowledgeFilesArchive starts a batch download and returns the
// server-suggested ZIP filename and a streaming reader. Callers MUST Close
// the returned reader. The request uses the streaming HTTP client so the
// default 30s timeout does not cut off large archives.
func (c *Client) OpenKnowledgeFilesArchive(ctx context.Context, knowledgeBaseID string, ids []string) (string, io.ReadCloser, error) {
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/batch-download", knowledgeBaseID)
resp, err := c.doRequestStream(ctx, http.MethodPost, path, BatchDownloadKnowledgeRequest{IDs: ids}, nil)
if err != nil {
return "", nil, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
return "", nil, newAPIError(resp.StatusCode, body)
}
filename := filenameFromContentDisposition(resp.Header.Get("Content-Disposition"))
return filename, resp.Body, nil
}
// filenameFromContentDisposition extracts the filename parameter from a
// Content-Disposition header. Returns "" on any parse failure or missing
// parameter — callers fall back to their own default in that case.

View File

@ -21,6 +21,7 @@
| POST | `/knowledge/:id/reparse` | 重新解析知识(异步) |
| POST | `/knowledge/:id/cancel-parse` | 取消正在进行的解析任务 |
| GET | `/knowledge/:id/download` | 下载原始文件(attachment) |
| POST | `/knowledge-bases/:id/knowledge/batch-download` | 将所选文档打包下载为 ZIP |
| GET | `/knowledge/:id/preview` | 内联预览文件(按扩展名设置 Content-Type) |
| PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 |
| PUT | `/knowledge/tags` | 批量更新知识标签 |
@ -712,7 +713,32 @@ curl --location --request POST 'http://localhost:8080/api/v1/knowledge/4c4e7c1a-
}
```
## GET `/knowledge/:id/download` - 下载原始文件
## 文件下载
### POST `/knowledge-bases/:id/knowledge/batch-download` - 批量下载
`POST /api/v1/knowledge-bases/:id/knowledge/batch-download`,请求体为 `{"ids":["文档ID1","文档ID2"]}`。
- 权限与单文件下载一致:JWT 用户需要 Contributor 身份及知识库 Editor 权限;API Key 需要 `retrieve` 能力或完整访问权限,并受知识库白名单限制。
- 每次最多 200 个 ID(重复 ID 只下载一次),原始文件实际读取字节合计最多 512 MiB。超限请分批下载。服务端同时最多处理 4 个批量下载请求,过多时返回 429。
- 成功返回 `application/zip` 附件;保留文件内容、中文文件名和知识库 `folder_path`。同名文件自动追加数字,移除不安全路径和 Windows 不支持的文件名字符。
- 手工编写的文档导出为 Markdown。没有原始文件的网页条目会被跳过;若跳过后没有任何可下载文件,返回 400。FAQ 条目通常没有原始文件,不能通过此接口导出。
- 任一条目无权访问、不属于当前知识库或读取失败时,整个请求失败,不返回残缺 ZIP。请求取消时返回 400,而不是 500。
- 压缩包在服务器临时目录生成后发送,请求结束或失败后清理临时文件;不包含分块、向量、标签、权限、Wiki 或图谱。
页面操作:进入知识库的文档页 → 点击「批量管理」→ 勾选文件,或「全选已加载」→「批量下载」。全选只包含列表已加载的文档;可先继续加载更多,再选择,单批仍受 200 项限制。没有原始文件的条目会自动跳过。ZIP 解压后可按文件夹批量上传到其他知识库。
**请求**:
```curl
curl --location --request POST 'http://localhost:8080/api/v1/knowledge-bases/kb-id/knowledge/batch-download' \
--header 'X-API-Key: sk-xxxxx' \
--header 'Content-Type: application/json' \
--data '{"ids":["文档ID1","文档ID2"]}' \
--output knowledge-files.zip
```
### GET `/knowledge/:id/download` - 单文件下载
以 `attachment` 方式下载知识对应的原始文件。

View File

@ -5193,6 +5193,91 @@ const docTemplate = `{
}
}
},
"/knowledge-bases/{id}/knowledge/batch-download": {
"post": {
"security": [
{
"Bearer": []
},
{
"ApiKeyAuth": []
}
],
"description": "将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。",
"consumes": [
"application/json"
],
"produces": [
"application/zip"
],
"tags": [
"知识管理"
],
"summary": "批量下载知识文件",
"parameters": [
{
"type": "string",
"description": "知识库ID",
"name": "id",
"in": "path",
"required": true
},
{
"description": "文档ID列表",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handler.BatchDownloadKnowledgeRequest"
}
}
],
"responses": {
"200": {
"description": "ZIP 压缩包",
"schema": {
"type": "file"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"429": {
"description": "Too Many Requests",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
}
}
}
},
"/knowledge-bases/{id}/knowledge/file": {
"post": {
"security": [
@ -23574,6 +23659,22 @@ const docTemplate = `{
}
}
},
"internal_handler.BatchDownloadKnowledgeRequest": {
"type": "object",
"required": [
"ids"
],
"properties": {
"ids": {
"type": "array",
"maxItems": 200,
"minItems": 1,
"items": {
"type": "string"
}
}
}
},
"internal_handler.CopyKnowledgeBaseRequest": {
"type": "object",
"required": [

View File

@ -5186,6 +5186,91 @@
}
}
},
"/knowledge-bases/{id}/knowledge/batch-download": {
"post": {
"security": [
{
"Bearer": []
},
{
"ApiKeyAuth": []
}
],
"description": "将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。",
"consumes": [
"application/json"
],
"produces": [
"application/zip"
],
"tags": [
"知识管理"
],
"summary": "批量下载知识文件",
"parameters": [
{
"type": "string",
"description": "知识库ID",
"name": "id",
"in": "path",
"required": true
},
{
"description": "文档ID列表",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/internal_handler.BatchDownloadKnowledgeRequest"
}
}
],
"responses": {
"200": {
"description": "ZIP 压缩包",
"schema": {
"type": "file"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"429": {
"description": "Too Many Requests",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError"
}
}
}
}
},
"/knowledge-bases/{id}/knowledge/file": {
"post": {
"security": [
@ -23567,6 +23652,22 @@
}
}
},
"internal_handler.BatchDownloadKnowledgeRequest": {
"type": "object",
"required": [
"ids"
],
"properties": {
"ids": {
"type": "array",
"maxItems": 200,
"minItems": 1,
"items": {
"type": "string"
}
}
}
},
"internal_handler.CopyKnowledgeBaseRequest": {
"type": "object",
"required": [

View File

@ -5030,6 +5030,17 @@ definitions:
- ids
- kb_id
type: object
internal_handler.BatchDownloadKnowledgeRequest:
properties:
ids:
items:
type: string
maxItems: 200
minItems: 1
type: array
required:
- ids
type: object
internal_handler.CopyKnowledgeBaseRequest:
properties:
source_id:
@ -9720,6 +9731,60 @@ paths:
summary: 获取知识列表
tags:
- 知识管理
/knowledge-bases/{id}/knowledge/batch-download:
post:
consumes:
- application/json
description: 将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。
parameters:
- description: 知识库ID
in: path
name: id
required: true
type: string
- description: 文档ID列表
in: body
name: request
required: true
schema:
$ref: '#/definitions/internal_handler.BatchDownloadKnowledgeRequest'
produces:
- application/zip
responses:
"200":
description: ZIP 压缩包
schema:
type: file
"400":
description: Bad Request
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
"403":
description: Forbidden
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
"404":
description: Not Found
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
"429":
description: Too Many Requests
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError'
security:
- Bearer: []
- ApiKeyAuth: []
summary: 批量下载知识文件
tags:
- 知识管理
/knowledge-bases/{id}/knowledge/file:
post:
consumes:

View File

@ -378,6 +378,15 @@ export function downKnowledgeDetails(id: string) {
return getDown(`/api/v1/knowledge/${id}/download`);
}
// 使用已有登录和租户请求头下载 ZIP,不将凭据放入下载链接。
export function batchDownloadKnowledge(kbId: string, ids: string[], signal?: AbortSignal): Promise<Blob> {
return post<Blob>(`/api/v1/knowledge-bases/${encodeURIComponent(kbId)}/knowledge/batch-download`, { ids }, {
responseType: 'blob',
timeout: 300000,
signal,
});
}
export function previewKnowledgeFile(id: string) {
return getDown(`/api/v1/knowledge/${id}/preview`);
}

View File

@ -710,6 +710,15 @@ export default {
selectedCount: '{count} selected',
clearSelection: 'Deselect all',
batchDelete: 'Delete selected',
batchDownload: 'Download selected',
batchDownloading: 'Preparing download…',
batchDownloadHint: 'Download a ZIP of up to 200 documents and 512 MiB of original content per batch. Select all includes loaded documents only; web pages without original files are skipped. The ZIP keeps knowledge-base folders.',
batchDownloadStarted: 'ZIP saving started. Extract it, then upload the files and folders.',
batchDownloadFailed: 'Batch download failed. Please try again.',
batchDownloadSkipped: 'Skipped {count} documents that have no original file.',
batchDownloadNoFiles: 'None of the selected documents have an original file to download.',
batchDownloadTooLarge: 'Selected files total more than 512 MiB. Choose fewer documents and try again.',
selectLoaded: 'Select loaded',
confirmBatchDeleteDocument: 'Delete {count} selected documents? This action cannot be undone.',
deleteSubmitted: 'Delete request submitted. Waiting for completion.',
deletePending: 'Deletion is still pending. Refresh later to check the result.',

View File

@ -710,6 +710,15 @@ export default {
selectedCount: '{count}件選択中',
clearSelection: 'すべて選択解除',
batchDelete: '選択項目を削除',
batchDownload: '選択項目をダウンロード',
batchDownloading: 'ダウンロードを準備中…',
batchDownloadHint: '1回につき最大200件、元ファイル合計512 MiBまでZIPでダウンロードできます。全選択は読み込み済み文書のみ対象で、元ファイルのないウェブページはスキップされます。ZIPはフォルダ構造を保持します。',
batchDownloadStarted: 'ZIPの保存を開始しました。展開してからファイルとフォルダをアップロードしてください。',
batchDownloadFailed: '一括ダウンロードに失敗しました。もう一度お試しください。',
batchDownloadSkipped: '元ファイルのない文書を{count}件スキップしました。',
batchDownloadNoFiles: '選択した文書にダウンロードできる元ファイルがありません。',
batchDownloadTooLarge: '選択したファイルの合計が512 MiBを超えています。件数を減らして分批ダウンロードしてください。',
selectLoaded: '読み込み済みを選択',
confirmBatchDeleteDocument: '選択した{count}件のドキュメントを削除しますか?この操作は取り消せません。',
deleteSubmitted: '削除リクエストを送信しました。完了までお待ちください。',
deletePending: '削除はまだ処理中です。後で更新して結果を確認してください。',

View File

@ -6736,6 +6736,15 @@ export default {
selectedCount: '{count}개 선택됨',
clearSelection: '선택 해제',
batchDelete: '선택 삭제',
batchDownload: '일괄 다운로드',
batchDownloading: '다운로드 준비 중…',
batchDownloadHint: '한 번에 최대 200개 문서, 원본 합계 512 MiB를 ZIP으로 다운로드합니다. 전체 선택은 불러온 문서만 포함하며, 원본이 없는 웹 페이지는 건너뜁니다. ZIP은 폴더 구조를 유지합니다.',
batchDownloadStarted: 'ZIP 저장을 시작했습니다. 압축을 푼 후 파일과 폴더를 업로드하세요.',
batchDownloadFailed: '일괄 다운로드에 실패했습니다. 다시 시도하세요.',
batchDownloadSkipped: '원본 파일이 없는 문서 {count}개를 건너뛰었습니다.',
batchDownloadNoFiles: '선택한 문서에 다운로드할 원본 파일이 없습니다.',
batchDownloadTooLarge: '선택한 파일 합계가 512 MiB를 초과합니다. 항목을 줄인 뒤 나눠 다운로드하세요.',
selectLoaded: '불러온 문서 선택',
confirmBatchDeleteDocument: '선택한 {count}개 문서를 삭제하시겠습니까? 삭제 후 복구할 수 없습니다.',
deleteSubmitted: '삭제 요청이 제출되었습니다. 완료를 기다리는 중입니다.',
deletePending: '삭제가 아직 완료되지 않았습니다. 잠시 후 새로고침하여 결과를 확인하세요.',

View File

@ -6736,6 +6736,15 @@ export default {
selectedCount: 'Выбрано: {count}',
clearSelection: 'Снять выделение',
batchDelete: 'Удалить выбранные',
batchDownload: 'Скачать выбранные',
batchDownloading: 'Подготовка архива…',
batchDownloadHint: 'ZIP: до 200 документов и 512 МиБ исходных файлов за раз. Выбор всех включает только загруженные документы; веб-страницы без исходных файлов пропускаются. ZIP сохраняет папки базы знаний.',
batchDownloadStarted: 'Началось сохранение ZIP. Распакуйте архив, затем загрузите файлы и папки.',
batchDownloadFailed: 'Не удалось скачать файлы. Повторите попытку.',
batchDownloadSkipped: 'Пропущено документов без исходного файла: {count}.',
batchDownloadNoFiles: 'У выбранных документов нет исходных файлов для скачивания.',
batchDownloadTooLarge: 'Суммарный размер выбранных файлов превышает 512 МиБ. Уменьшите выбор и скачайте частями.',
selectLoaded: 'Выбрать загруженные',
confirmBatchDeleteDocument: 'Удалить {count} выбранных документов? Это действие нельзя отменить.',
deleteSubmitted: 'Запрос на удаление отправлен. Ожидание завершения.',
deletePending: 'Удаление ещё не завершено. Обновите страницу позже, чтобы проверить результат.',

View File

@ -6738,6 +6738,15 @@ export default {
selectedCount: '已选 {count} 项',
clearSelection: '取消选择',
batchDelete: '批量删除',
batchDownload: '批量下载',
batchDownloading: '正在打包下载…',
batchDownloadHint: '下载所选文件为 ZIP,每批最多 200 项、原始文件合计 512 MiB。全选仅包含已加载的文档;没有原始文件的网页条目会自动跳过。ZIP 会保留知识库文件夹。',
batchDownloadStarted: '已开始保存 ZIP,解压后即可按文件夹批量上传文件',
batchDownloadFailed: '批量下载失败,请稍后重试',
batchDownloadSkipped: '已跳过 {count} 个没有原始文件的文档',
batchDownloadNoFiles: '所选文档没有可下载的原始文件',
batchDownloadTooLarge: '所选文件合计超过 512 MiB,请减少选择后分批下载',
selectLoaded: '全选已加载',
confirmBatchDeleteDocument: '确认删除选中的 {count} 个文档?删除后将无法恢复。',
deleteSubmitted: '删除任务已提交,正在等待完成。',
deletePending: '删除尚未完成,请稍后刷新查看结果。',

View File

@ -137,6 +137,19 @@ instance.interceptors.response.use(
if (!error.response) {
return Promise.reject({ message: t('error.networkError') });
}
// 文件下载失败时服务端仍返回 JSON;先还原错误信息,避免被 Blob 隐藏。
// 不依赖 Content-Type:网关可能把错误改成 text/plain 或空类型。
if (typeof Blob !== 'undefined' && error.response.data instanceof Blob) {
try {
const text = (await error.response.data.text()).trim();
if (text.startsWith('{') || text.startsWith('[') || error.response.data.type.includes('json')) {
error.response.data = JSON.parse(text);
}
} catch {
// 非法 JSON 继续使用原有错误处理。
}
}
// 公开接口(auto-setup / login / register / oidc)的 401 不走 refresh 逻辑,直接返回错误
if ((error.response.status === 401 || error.response.status === 403) && isPublicAuthRequest(originalRequest?.url)) {

View File

@ -32,6 +32,7 @@ import {
reparseKnowledge,
cancelKnowledgeParse,
batchDeleteKnowledge,
batchDownloadKnowledge,
delKnowledgeDetails,
batchReparseKnowledge,
getKnowledgeSpans,
@ -42,6 +43,7 @@ import {
downKnowledgeDetails,
type KnowledgeFolderTree,
} from "@/api/knowledge-base/index";
import { isBatchDownloadableKnowledge } from './knowledgeDownloadFileName';
import { waitForKnowledgeDeletion } from '@/utils/knowledgeDeletion';
import { knowledgeSpansPayloadHasTrace } from '@/utils/knowledgeTrace';
import FAQEntryManager from './components/FAQEntryManager.vue';
@ -441,6 +443,67 @@ let lastSelectedIndex = -1;
const batchDeleting = ref(false);
const batchReparsing = ref(false);
const batchTagging = ref(false);
const batchDownloading = ref(false);
let batchDownloadController: AbortController | undefined;
const MAX_BATCH_DOWNLOAD_FILES = 200;
const MAX_BATCH_DOWNLOAD_BYTES = 512 * 1024 * 1024;
const handleBatchDownload = async () => {
if (batchDownloading.value || batchDeleting.value || batchReparsing.value || batchTagging.value || selectedIds.value.size === 0) return;
const selected = Array.from(selectedIds.value);
const itemsById = new Map((cardList.value || []).map((item) => [item.id, item]));
const ids = selected.filter((id) => isBatchDownloadableKnowledge(itemsById.get(id)));
const skipped = selected.length - ids.length;
if (ids.length === 0) {
MessagePlugin.warning(t('knowledgeBase.batchDownloadNoFiles'));
return;
}
if (ids.length > MAX_BATCH_DOWNLOAD_FILES) {
MessagePlugin.warning(t('knowledgeBase.batchDownloadHint'));
return;
}
let knownBytes = 0;
for (const id of ids) {
const size = Number(itemsById.get(id)?.file_size);
if (Number.isFinite(size) && size > 0) knownBytes += size;
}
if (knownBytes > MAX_BATCH_DOWNLOAD_BYTES) {
MessagePlugin.warning(t('knowledgeBase.batchDownloadTooLarge'));
return;
}
if (skipped > 0) {
MessagePlugin.warning(t('knowledgeBase.batchDownloadSkipped', { count: skipped }));
}
const controller = new AbortController();
batchDownloadController = controller;
batchDownloading.value = true;
try {
const blob = await batchDownloadKnowledge(kbId.value, ids, controller.signal);
if (controller.signal.aborted) return;
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.download = `knowledge-files-${Date.now()}.zip`;
document.body.appendChild(link);
link.click();
link.remove();
// 延迟释放,给浏览器留出开始保存文件的时间。
window.setTimeout(() => URL.revokeObjectURL(url), 60000);
MessagePlugin.success(t('knowledgeBase.batchDownloadStarted'));
} catch (error: any) {
if (!controller.signal.aborted) {
MessagePlugin.error(error?.message || t('knowledgeBase.batchDownloadFailed'));
}
} finally {
if (batchDownloadController === controller) {
batchDownloading.value = false;
batchDownloadController = undefined;
}
}
};
watch(kbId, () => batchDownloadController?.abort());
onUnmounted(() => batchDownloadController?.abort());
const batchTagDialogVisible = ref(false);
const batchTagPreSelectedIds = computed(() => {
const ids = Array.from(selectedIds.value);
@ -497,7 +560,7 @@ const awaitBatchReparseReflection = async (ids: string[]) => {
};
const confirmBatchReparse = async () => {
if (batchReparsing.value || batchDeleting.value || selectedIds.value.size === 0) return;
if (batchReparsing.value || batchDeleting.value || batchDownloading.value || selectedIds.value.size === 0) return;
const allIds = Array.from(selectedIds.value);
const ids = allIds.filter((id) => {
const item = cardList.value.find((c) => c.id === id);
@ -2062,7 +2125,7 @@ const {
itemSelector: '.knowledge-card[data-select-id], .doc-list-row[data-select-id]',
selectedIds,
getItemId: (el) => el.dataset.selectId || null,
enabled: computed(() => canEdit.value && !isFAQ.value && cardList.value.length > 0),
enabled: computed(() => (canEdit.value || canDownloadKnowledge.value) && !isFAQ.value && cardList.value.length > 0),
onSelectionStart: () => {
batchMode.value = true;
},
@ -2096,7 +2159,7 @@ const deleteKnowledgeDocuments = async (
submit: () => Promise<any>,
batch: boolean,
) => {
if (batchDeleting.value || batchReparsing.value || ids.length === 0) return;
if (batchDeleting.value || batchReparsing.value || batchDownloading.value || ids.length === 0) return;
const targetKbId = kbId.value;
const generation = ++deleteGeneration;
const isActive = () => generation === deleteGeneration && isCurrentKb(targetKbId);
@ -2163,7 +2226,7 @@ const confirmBatchDelete = () => {
};
const handleBatchTag = () => {
if (batchDeleting.value || batchReparsing.value || batchTagging.value || selectedIds.value.size === 0) return;
if (batchDeleting.value || batchReparsing.value || batchTagging.value || batchDownloading.value || selectedIds.value.size === 0) return;
batchTagDialogVisible.value = true;
};
@ -2594,6 +2657,12 @@ async function createNewSession(value: string): Promise<void> {
</div>
</div>
<div class="doc-filter-bar__trailing">
<t-button v-if="(canDownloadKnowledge || canMutateKnowledge) && cardList.length"
variant="outline" size="small"
:disabled="batchDeleting || batchReparsing || batchTagging || batchDownloading"
@click="toggleBatchMode">
{{ $t(batchMode ? 'knowledgeBase.clearSelection' : 'menu.batchManage') }}
</t-button>
<div class="doc-view-toggle" role="group" :aria-label="$t('knowledgeBase.viewModeToggle')">
<t-tooltip :content="$t('knowledgeBase.viewModeGrid')" placement="top">
<button type="button" class="doc-view-toggle-btn" :class="{ active: viewMode === 'grid' }"
@ -2710,10 +2779,12 @@ async function createNewSession(value: string): Promise<void> {
</div>
<div class="doc-batch-bar-anchor" v-show="batchMode || selectedIds.size > 0">
<DocumentBatchBar :count="selectedIds.size" :delete-loading="batchDeleting"
:reparse-loading="batchReparsing" :tag-loading="batchTagging" :visible="batchMode || selectedIds.size > 0"
:reparse-loading="batchReparsing" :tag-loading="batchTagging" :download-loading="batchDownloading"
:can-download="canDownloadKnowledge" :can-mutate="canMutateKnowledge"
:visible="batchMode || selectedIds.size > 0"
:show-move-to-folder="canEdit" :folder-options="folderOptions"
@cancel="handleBatchCancel" @delete="confirmBatchDelete" @reparse="confirmBatchReparse"
@batch-tag="handleBatchTag"
@batch-tag="handleBatchTag" @download="handleBatchDownload" @select-loaded="toggleSelectAll(true)"
@move-to-folder="(path: string) => moveKnowledgeIntoFolder(Array.from(selectedIds), path)" />
</div>
</div>

View File

@ -108,7 +108,7 @@ const fileName = computed(() => props.item.file_name || props.item.title || prop
</div>
<!-- 批量管理 -->
<div v-if="canMutateKnowledge" class="doc-action-menu-item" @click.stop="emit('batch-manage')">
<div v-if="canMutateKnowledge || canDownload" class="doc-action-menu-item" @click.stop="emit('batch-manage')">
<t-icon class="icon" name="queue" />
<span>{{ $t('menu.batchManage') }}</span>
</div>

View File

@ -8,6 +8,9 @@ defineProps<{
deleteLoading?: boolean;
reparseLoading?: boolean;
tagLoading?: boolean;
downloadLoading?: boolean;
canDownload?: boolean;
canMutate?: boolean;
// When true the bar stays visible even with 0 selections, so users can exit
// batch mode from here without selecting anything first.
visible?: boolean;
@ -22,6 +25,8 @@ const emit = defineEmits<{
(e: 'reparse'): void;
(e: 'batchTag'): void;
(e: 'moveToFolder', folderPath: string): void;
(e: 'download'): void;
(e: 'selectLoaded'): void;
}>();
const { t } = useI18n();
@ -36,32 +41,48 @@ const folderPickerVisible = ref(false);
<div class="batch-bar-inner">
<div class="batch-bar-left">
<span class="batch-bar-count">{{ t('knowledgeBase.selectedCount', { count }) }}</span>
<t-button variant="text" theme="default" size="small" class="batch-bar-clear" @click="emit('cancel')">
<t-button variant="text" theme="default" size="small" class="batch-bar-clear"
:disabled="downloadLoading" @click="emit('selectLoaded')">
{{ t('knowledgeBase.selectLoaded') }}
</t-button>
<t-button variant="text" theme="default" size="small" class="batch-bar-clear"
:disabled="downloadLoading" @click="emit('cancel')">
{{ t('knowledgeBase.clearSelection') }}
</t-button>
</div>
<div class="batch-bar-actions">
<t-popconfirm theme="warning" :content="t('knowledgeBase.confirmBatchReparseDocument', { count })"
<t-tooltip v-if="canDownload" :content="t('knowledgeBase.batchDownloadHint')">
<span class="batch-download-trigger">
<t-button theme="primary" size="small" :loading="downloadLoading"
:disabled="count === 0 || count > 200 || deleteLoading || reparseLoading || tagLoading || downloadLoading"
@click="emit('download')">
<template #icon><t-icon name="download" size="14px" /></template>
{{ t(downloadLoading ? 'knowledgeBase.batchDownloading' : 'knowledgeBase.batchDownload') }}
</t-button>
</span>
</t-tooltip>
<t-popconfirm v-if="canMutate" theme="warning" :content="t('knowledgeBase.confirmBatchReparseDocument', { count })"
:confirm-btn="{ content: t('knowledgeBase.confirmBatchReparse'), theme: 'warning' }"
:cancel-btn="{ content: t('common.cancel') }" placement="top" @confirm="emit('reparse')">
<t-button theme="default" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading" :loading="reparseLoading" @click.stop>
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading" :loading="reparseLoading" @click.stop>
<template #icon><t-icon name="refresh" size="14px" /></template>
{{ t('knowledgeBase.rebuildDocument') }}
</t-button>
</t-popconfirm>
<t-button theme="default" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading" :loading="tagLoading"
<t-button v-if="canMutate" theme="default" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading" :loading="tagLoading"
@click="emit('batchTag')">
<template #icon><t-icon name="discount" size="14px" /></template>
{{ t('knowledgeBase.batchTag') }}
</t-button>
<t-popup v-if="showMoveToFolder" v-model:visible="folderPickerVisible" trigger="click"
<t-popup v-if="canMutate && showMoveToFolder" v-model:visible="folderPickerVisible" trigger="click"
placement="top" overlay-class-name="card-more" destroy-on-close>
<t-button theme="default" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading">
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading">
<template #icon><t-icon name="folder" size="14px" /></template>
{{ t('knowledgeBase.moveToFolder.action') }}
</t-button>
@ -73,11 +94,11 @@ const folderPickerVisible = ref(false);
</template>
</t-popup>
<t-popconfirm theme="warning" :content="t('knowledgeBase.confirmBatchDeleteDocument', { count })"
<t-popconfirm v-if="canMutate" theme="warning" :content="t('knowledgeBase.confirmBatchDeleteDocument', { count })"
:confirm-btn="{ content: t('knowledgeBase.confirmDelete'), theme: 'danger' }"
:cancel-btn="{ content: t('common.cancel') }" placement="top" @confirm="emit('delete')">
<t-button theme="danger" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading" :loading="deleteLoading" @click.stop>
:disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading" :loading="deleteLoading" @click.stop>
<template #icon><t-icon name="delete" size="14px" /></template>
{{ t('knowledgeBase.batchDelete') }}
</t-button>
@ -93,7 +114,7 @@ const folderPickerVisible = ref(false);
position: relative;
z-index: 5;
width: 100%;
max-width: 560px;
max-width: 920px;
margin: 0 auto;
padding: 0 4px;
box-sizing: border-box;
@ -101,6 +122,7 @@ const folderPickerVisible = ref(false);
.batch-bar-inner {
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: space-between;
gap: 12px;
@ -113,6 +135,7 @@ const folderPickerVisible = ref(false);
.batch-bar-left {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 4px;
min-width: 0;
@ -147,6 +170,10 @@ const folderPickerVisible = ref(false);
gap: 8px;
}
.batch-download-trigger {
display: inline-flex;
}
.batch-bar-fade-enter-active,
.batch-bar-fade-leave-active {
transition: transform 0.2s ease, opacity 0.2s ease;

View File

@ -0,0 +1,35 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { readFileSync } from 'node:fs'
const knowledgeBase = readFileSync(new URL('../KnowledgeBase.vue', import.meta.url), 'utf8')
const batchBar = readFileSync(new URL('./DocumentBatchBar.vue', import.meta.url), 'utf8')
const listView = readFileSync(new URL('./DocumentListView.vue', import.meta.url), 'utf8')
const knowledgeApi = readFileSync(new URL('../../../api/knowledge-base/index.ts', import.meta.url), 'utf8')
const request = readFileSync(new URL('../../../utils/request.ts', import.meta.url), 'utf8')
test('批量下载使用受认证的 Blob 请求并支持取消', () => {
assert.match(knowledgeApi, /knowledge-bases\/\$\{encodeURIComponent\(kbId\)\}\/knowledge\/batch-download/)
assert.match(knowledgeApi, /responseType:\s*'blob'/)
assert.match(knowledgeApi, /signal,/)
assert.match(knowledgeBase, /new AbortController\(\)/)
assert.match(knowledgeBase, /batchDownloadController\?\.abort\(\)/)
})
test('批量下载界面限制单批 200 项并保留只读权限边界', () => {
assert.match(batchBar, /v-if="canDownload"/)
assert.match(batchBar, /count > 200/)
assert.match(batchBar, /batch-download-trigger/)
assert.match(batchBar, /v-if="canMutate"/)
assert.match(knowledgeBase, /isBatchDownloadableKnowledge/)
assert.match(knowledgeBase, /ids\.length > MAX_BATCH_DOWNLOAD_FILES/)
assert.match(knowledgeBase, /:can-download="canDownloadKnowledge"/)
assert.match(knowledgeBase, /@select-loaded="toggleSelectAll\(true\)"/)
assert.match(listView, /v-if="canEdit \|\| canDownload"/)
})
test('Blob 格式的 JSON 错误会还原为可读消息', () => {
assert.match(request, /error\.response\.data instanceof Blob/)
assert.match(request, /JSON\.parse\(text\)/)
assert.match(request, /text\.startsWith\('\{'\)/)
})

View File

@ -348,7 +348,7 @@ const handleAction = (action: 'download' | 'edit' | 'view-trace' | 'reparse' | '
>
<div class="card-content">
<div class="card-content-nav">
<div v-if="canEdit && batchMode" class="card-nav-check" @click.stop>
<div v-if="(canEdit || canDownload) && batchMode" class="card-nav-check" @click.stop>
<t-checkbox
class="card-select-checkbox"
size="small"

View File

@ -260,7 +260,7 @@ const handleAction = (action: 'download' | 'edit' | 'reparse' | 'cancel-parse' |
<div ref="stickySentinel" class="doc-list-sticky-sentinel" aria-hidden="true"></div>
<div class="doc-list-header" :class="{ 'is-stuck': headerStuck }" role="row">
<div class="cell cell-check" role="columnheader" @click.stop>
<t-checkbox class="doc-list-check" size="small" :checked="allSelected" :indeterminate="someSelected"
<t-checkbox v-if="canEdit || canDownload" class="doc-list-check" size="small" :checked="allSelected" :indeterminate="someSelected"
:disabled="!items.length" :title="t('knowledgeBase.selectAll')" @change="onHeaderCheckboxChange" />
</div>
<div class="cell cell-name" role="columnheader">{{ t('knowledgeBase.columnName') }}</div>
@ -306,7 +306,7 @@ const handleAction = (action: 'download' | 'edit' | 'reparse' | 'cancel-parse' |
:class="{ selected: selectedIds.has(item.id), 'menu-open': moreOpen === item.id }" :data-select-id="item.id"
role="row" @click="emit('open', item)">
<div class="cell cell-check" @click.stop>
<t-checkbox class="doc-list-check" size="small" :checked="selectedIds.has(item.id)" :title="item.file_name"
<t-checkbox v-if="canEdit || canDownload" class="doc-list-check" size="small" :checked="selectedIds.has(item.id)" :title="item.file_name"
@change="(c: boolean, ctx?: { e?: Event }) => onRowCheckboxChange(item, c, ctx)" />
</div>

View File

@ -1,7 +1,7 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { resolveKnowledgeDownloadFileName } from './knowledgeDownloadFileName.ts';
import { isBatchDownloadableKnowledge, resolveKnowledgeDownloadFileName } from './knowledgeDownloadFileName.ts';
test('prefers the original filename over the extensionless display name', () => {
assert.equal(resolveKnowledgeDownloadFileName({
@ -20,6 +20,14 @@ test('falls back to the backend filename when no original filename is present',
}), 'notes.txt');
});
test('batch download skips web pages without an original file', () => {
assert.equal(isBatchDownloadableKnowledge({ type: 'manual' }), true);
assert.equal(isBatchDownloadableKnowledge({ type: 'file', file_path: 'stored' }), true);
assert.equal(isBatchDownloadableKnowledge({ type: 'url' }), false);
assert.equal(isBatchDownloadableKnowledge({ type: 'url', file_path: 'snapshot.html' }), true);
assert.equal(isBatchDownloadableKnowledge(undefined), false);
});
test('adds the markdown extension to manual documents exactly once', () => {
assert.equal(resolveKnowledgeDownloadFileName({
id: 'knowledge-3',

View File

@ -13,3 +13,9 @@ export function resolveKnowledgeDownloadFileName(item: KnowledgeDownloadItem): s
}
return baseName;
}
export function isBatchDownloadableKnowledge(item?: { type?: string; file_path?: string } | null): boolean {
if (!item) return false;
if (item.type === 'manual') return true;
return Boolean(item.file_path);
}

View File

@ -0,0 +1,361 @@
package handler
import (
"archive/zip"
"context"
stderrors "errors"
"fmt"
"io"
"net/http"
"os"
"path"
"strings"
"time"
"unicode"
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/filetransport"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/gin-gonic/gin"
)
const (
maxBatchDownloadFiles = 200
maxBatchDownloadBytes int64 = 512 * 1024 * 1024
maxConcurrentBatchDownloads = 4
)
var batchDownloadSlots = make(chan struct{}, maxConcurrentBatchDownloads)
// BatchDownloadKnowledgeRequest 指定同一知识库中需要下载的文档。
type BatchDownloadKnowledgeRequest struct {
IDs []string `json:"ids" binding:"required,min=1,max=200,dive,required,max=128"`
}
type knowledgeDownloadEntry struct {
ID string
FolderPath string
}
// BatchDownloadKnowledge godoc
// @Summary 批量下载知识文件
// @Description 将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。
// @Tags 知识管理
// @Accept json
// @Produce application/zip
// @Param id path string true "知识库ID"
// @Param request body BatchDownloadKnowledgeRequest true "文档ID列表"
// @Success 200 {file} file "ZIP 压缩包"
// @Failure 400 {object} errors.AppError
// @Failure 401 {object} errors.AppError
// @Failure 403 {object} errors.AppError
// @Failure 404 {object} errors.AppError
// @Failure 429 {object} errors.AppError
// @Failure 500 {object} errors.AppError
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/batch-download [post]
func (h *KnowledgeHandler) BatchDownloadKnowledge(c *gin.Context) {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64*1024)
var req BatchDownloadKnowledgeRequest
if err := c.ShouldBindJSON(&req); err != nil {
_ = c.Error(errors.NewBadRequestError("请选择 1~200 个文档进行批量下载"))
return
}
kbID := c.Param("id")
grant, err := resolveHandlerKBAccessFor(
c, kbID, h.kbService, h.kbShareService, h.agentShareService, types.OrgRoleEditor,
)
if err != nil {
_ = c.Error(err)
return
}
tenantID := grant.EffectiveTenantID
ctx := types.WithExecutionTenant(c.Request.Context(), tenantID)
ids := uniqueKnowledgeDownloadIDs(req.IDs)
if len(ids) == 0 {
_ = c.Error(errors.NewBadRequestError("文档 ID 不能为空"))
return
}
items, err := h.kgService.GetKnowledgeBatch(ctx, tenantID, ids)
if err != nil {
_ = c.Error(errors.NewInternalServerError("无法获取待下载文档,请稍后重试"))
return
}
byID := make(map[string]*types.Knowledge, len(items))
for _, item := range items {
if item != nil && item.KnowledgeBaseID == kbID && item.TenantID == tenantID {
byID[item.ID] = item
}
}
entries := make([]knowledgeDownloadEntry, 0, len(ids))
// 在读取任何文件之前检查整批文档,防止混入其他知识库或租户的 ID。
for _, id := range ids {
item := byID[id]
if item == nil {
_ = c.Error(errors.NewNotFoundError("部分文档不存在或不属于当前知识库,请刷新列表后重试"))
return
}
if !item.IsManual() && item.FilePath == "" {
continue
}
entries = append(entries, knowledgeDownloadEntry{ID: id, FolderPath: item.FolderPath})
}
if len(entries) == 0 {
_ = c.Error(errors.NewBadRequestError("所选文档没有可下载的原始文件,请取消选择后重试"))
return
}
if !tryAcquireBatchDownloadSlot() {
_ = c.Error(errors.NewTooManyRequestsError("当前批量下载过多,请稍后重试"))
return
}
defer releaseBatchDownloadSlot()
// 先在临时文件中完整生成压缩包,避免读取失败时向用户返回残缺 ZIP。
archive, err := os.CreateTemp("", "weknora-download-*.zip")
if err != nil {
_ = c.Error(errors.NewInternalServerError("无法创建下载压缩包,请稍后重试"))
return
}
archivePath := archive.Name()
defer func() {
_ = archive.Close()
_ = os.Remove(archivePath)
}()
if err := writeKnowledgeDownloadArchive(
ctx, archive, entries, h.kgService.GetKnowledgeFile, maxBatchDownloadBytes,
); err != nil {
mapped := mapKnowledgeDownloadError(err)
if appErr, ok := errors.IsAppError(mapped); !ok || appErr.HTTPCode >= 500 {
logger.ErrorWithFields(ctx, err, nil)
}
_ = c.Error(mapped)
return
}
if _, err := archive.Seek(0, io.SeekStart); err != nil {
_ = c.Error(errors.NewInternalServerError("无法读取下载压缩包,请稍后重试"))
return
}
filename := "knowledge-files-" + time.Now().Format("20060102-150405") + ".zip"
if err := filetransport.Serve(c.Writer, c.Request, archive, filetransport.Options{
Filename: filename, Download: true, ContentType: "application/zip", CacheControl: "private, no-store",
}); err != nil {
logger.Errorf(ctx, "Failed to send knowledge archive: %v", err)
}
}
func tryAcquireBatchDownloadSlot() bool {
select {
case batchDownloadSlots <- struct{}{}:
return true
default:
return false
}
}
func releaseBatchDownloadSlot() {
<-batchDownloadSlots
}
func uniqueKnowledgeDownloadIDs(input []string) []string {
ids := make([]string, 0, len(input))
seen := make(map[string]bool, len(input))
for _, id := range input {
id = strings.TrimSpace(id)
if id != "" && !seen[id] {
seen[id] = true
ids = append(ids, id)
}
}
return ids
}
func isKnowledgeDownloadCanceled(err error) bool {
return err != nil && (stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded))
}
func mapKnowledgeDownloadError(err error) error {
if err == nil {
return nil
}
if _, ok := errors.IsAppError(err); ok {
return err
}
if isKnowledgeDownloadCanceled(err) {
return errors.NewBadRequestError("下载已取消")
}
return err
}
type knowledgeDownloadOpener func(context.Context, string) (io.ReadCloser, string, error)
func writeKnowledgeDownloadArchive(
ctx context.Context,
output io.Writer,
entries []knowledgeDownloadEntry,
open knowledgeDownloadOpener,
limit int64,
) error {
writer := zip.NewWriter(output)
usedNames := make(map[string]bool, len(entries))
var total int64
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return err
}
file, filename, err := open(ctx, entry.ID)
if err != nil {
if isKnowledgeDownloadCanceled(err) {
return err
}
return errors.NewInternalServerError("部分文件无法读取,未生成压缩包,请检查文档后重试").WithDetails(
gin.H{"knowledge_id": entry.ID},
)
}
name := uniqueKnowledgeDownloadZipPath(entry.FolderPath, filename, usedNames)
// 原样存储可避免 PDF、Office 等已压缩文件再次压缩的 CPU 开销。
zipEntry, err := writer.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
if err != nil {
_ = file.Close()
return errors.NewInternalServerError("无法写入下载压缩包,请稍后重试")
}
n, copyErr := io.Copy(zipEntry, io.LimitReader(
&knowledgeDownloadReader{ctx: ctx, reader: file}, limit-total+1,
))
closeErr := file.Close()
total += n
if total > limit {
return errors.NewBadRequestError("所选文件合计超过 512 MiB,请分批下载")
}
if isKnowledgeDownloadCanceled(copyErr) {
return copyErr
}
if copyErr != nil || closeErr != nil {
return errors.NewInternalServerError("部分文件读取失败,未生成压缩包,请稍后重试").WithDetails(
gin.H{"knowledge_id": entry.ID},
)
}
}
if err := writer.Close(); err != nil {
return errors.NewInternalServerError("无法完成下载压缩包,请稍后重试")
}
return nil
}
// 每次读取前检查取消信号,用户离开页面后停止继续打包。
type knowledgeDownloadReader struct {
ctx context.Context
reader io.Reader
}
func (r *knowledgeDownloadReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
func uniqueKnowledgeDownloadName(filename string, used map[string]bool) string {
return uniqueKnowledgeDownloadZipPath("", filename, used)
}
func uniqueKnowledgeDownloadZipPath(folderPath, filename string, used map[string]bool) string {
base := sanitizeDownloadFileName(filename)
ext := path.Ext(base)
stem := strings.TrimSuffix(base, ext)
dir := sanitizeDownloadFolderPath(folderPath)
name := base
if dir != "" {
name = dir + "/" + base
}
for index := 2; used[strings.ToLower(name)]; index++ {
suffixed := fmt.Sprintf("%s (%d)%s", stem, index, ext)
if dir != "" {
name = dir + "/" + suffixed
} else {
name = suffixed
}
}
used[strings.ToLower(name)] = true
return name
}
func sanitizeDownloadFileName(filename string) string {
filename = path.Base(strings.ReplaceAll(filename, "\\", "/"))
filename = sanitizeDownloadRunes(filename)
filename = strings.Trim(filename, " .")
if filename == "" {
filename = "document"
}
ext := path.Ext(filename)
if len(ext) > 20 {
ext = ""
}
stem := strings.TrimSuffix(filename, ext)
stem = shortenDownloadName(stem, 180)
stem = strings.TrimRight(stem, " .")
if stem == "" {
stem = "document"
}
return escapeWindowsReservedName(stem) + ext
}
func sanitizeDownloadFolderPath(folderPath string) string {
folderPath = strings.ReplaceAll(folderPath, "\\", "/")
var parts []string
var total int
for _, part := range strings.Split(folderPath, "/") {
part = sanitizeDownloadRunes(part)
part = strings.Trim(part, " .")
if part == "" || part == "." || part == ".." {
continue
}
part = shortenDownloadName(part, 80)
part = strings.TrimRight(part, " .")
if part == "" {
continue
}
part = escapeWindowsReservedName(part)
if total+len(part)+1 > 160 {
break
}
parts = append(parts, part)
total += len(part) + 1
}
return strings.Join(parts, "/")
}
func sanitizeDownloadRunes(value string) string {
return strings.Map(func(r rune) rune {
if unicode.IsControl(r) || strings.ContainsRune(`<>:"/\|?*`, r) {
return '_'
}
return r
}, value)
}
func shortenDownloadName(value string, limit int) string {
var shortened strings.Builder
for _, r := range value {
if shortened.Len()+len(string(r)) > limit {
break
}
shortened.WriteRune(r)
}
return shortened.String()
}
func escapeWindowsReservedName(stem string) string {
reserved := strings.ToUpper(strings.Split(stem, ".")[0])
reservedRunes := []rune(reserved)
if reserved == "CON" || reserved == "PRN" || reserved == "AUX" || reserved == "NUL" ||
(len(reservedRunes) == 4 && (strings.HasPrefix(reserved, "COM") || strings.HasPrefix(reserved, "LPT")) &&
strings.ContainsRune("123456789¹²³", reservedRunes[3])) {
return "_" + stem
}
return stem
}

View File

@ -0,0 +1,393 @@
package handler
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"unicode/utf8"
apperrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/middleware"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
type downloadKnowledgeStub struct {
interfaces.KnowledgeService
items []*types.Knowledge
names map[string]string
opened []string
failID string
expectedTenant uint64
}
func (s *downloadKnowledgeStub) GetKnowledgeBatch(context.Context, uint64, []string) ([]*types.Knowledge, error) {
return s.items, nil
}
func (s *downloadKnowledgeStub) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) {
want := s.expectedTenant
if want == 0 {
want = 7
}
if types.MustTenantIDFromContext(ctx) != want {
return nil, "", fmt.Errorf("错误的租户上下文")
}
s.opened = append(s.opened, id)
if s.failID == id {
return nil, "", fmt.Errorf("模拟存储读取失败")
}
return io.NopCloser(strings.NewReader("原文-" + id)), s.names[id], nil
}
type downloadKBStub struct {
interfaces.KnowledgeBaseService
kb *types.KnowledgeBase
}
func (s *downloadKBStub) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) {
return s.kb, nil
}
type downloadShareStub struct {
interfaces.KBShareService
permission types.OrgMemberRole
}
func (s *downloadShareStub) CheckTenantKBPermission(
context.Context, string, uint64, types.TenantRole,
) (types.OrgMemberRole, bool, error) {
return s.permission, true, nil
}
func runBatchDownload(
t *testing.T,
svc *downloadKnowledgeStub,
ids []string,
kb *types.KnowledgeBase,
share interfaces.KBShareService,
scope *types.TenantAPIKeyScope,
) *httptest.ResponseRecorder {
t.Helper()
gin.SetMode(gin.TestMode)
router := gin.New()
router.Use(middleware.ErrorHandler(), func(c *gin.Context) {
ctx := types.WithCaller(c.Request.Context(), types.Caller{
TenantID: 7, UserID: "user-1", Role: types.TenantRoleContributor,
})
ctx = types.WithExecutionTenant(ctx, 7)
if scope != nil {
ctx = types.WithTenantAPIKeyScope(ctx, *scope)
}
c.Request = c.Request.WithContext(ctx)
c.Set(types.TenantIDContextKey.String(), uint64(7))
c.Set(types.UserIDContextKey.String(), "user-1")
c.Next()
})
h := &KnowledgeHandler{
kgService: svc,
kbService: &downloadKBStub{kb: kb},
kbShareService: share,
}
router.POST("/knowledge-bases/:id/knowledge/batch-download", h.BatchDownloadKnowledge)
body, err := json.Marshal(BatchDownloadKnowledgeRequest{IDs: ids})
require.NoError(t, err)
req := httptest.NewRequest(
http.MethodPost, "/knowledge-bases/kb-1/knowledge/batch-download", bytes.NewReader(body),
)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
return w
}
func TestBatchDownloadKnowledgeProducesCompleteZIP(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
svc := &downloadKnowledgeStub{
items: []*types.Knowledge{
{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-a"},
{ID: "b", TenantID: 7, KnowledgeBaseID: "kb-1", Type: types.KnowledgeTypeManual},
{ID: "c", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-c"},
},
names: map[string]string{"a": "设计说明.md", "b": "设计说明.md", "c": `..\设计说明 (2).md`},
}
w := runBatchDownload(t, svc, []string{"a", "b", "a", "c"},
&types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil)
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
require.Equal(t, "application/zip", w.Header().Get("Content-Type"))
require.Equal(t, "private, no-store", w.Header().Get("Cache-Control"))
require.Contains(t, w.Header().Get("Content-Disposition"), "attachment")
reader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
require.NoError(t, err)
require.Len(t, reader.File, 3)
for index, name := range []string{"设计说明.md", "设计说明 (2).md", "设计说明 (2) (2).md"} {
entry := reader.File[index]
require.Equal(t, name, entry.Name)
file, err := entry.Open()
require.NoError(t, err)
content, err := io.ReadAll(file)
require.NoError(t, err)
require.NoError(t, file.Close())
require.Equal(t, "原文-"+[]string{"a", "b", "c"}[index], string(content))
}
require.Equal(t, []string{"a", "b", "c"}, svc.opened)
entries, err := os.ReadDir(os.TempDir())
require.NoError(t, err)
require.Empty(t, entries, "请求完成后不应留下临时压缩包")
}
func TestBatchDownloadKnowledgeRejectsInvalidSelectionsBeforeReading(t *testing.T) {
cases := []struct {
name string
ids []string
item *types.Knowledge
kb *types.KnowledgeBase
share interfaces.KBShareService
scope *types.TenantAPIKeyScope
status int
}{
{
name: "空列表", status: 400,
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "空白ID", ids: []string{" "}, status: 400,
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "超出数量", ids: make([]string, maxBatchDownloadFiles+1), status: 400,
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "文档缺失", ids: []string{"missing"}, status: 404,
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "跨知识库", ids: []string{"a"}, status: 404,
item: &types.Knowledge{ID: "a", TenantID: 7, KnowledgeBaseID: "other", FilePath: "secret"},
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "跨租户", ids: []string{"a"}, status: 404,
item: &types.Knowledge{ID: "a", TenantID: 8, KnowledgeBaseID: "kb-1", FilePath: "secret"},
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "仅无原文件", ids: []string{"a"}, status: 400,
item: &types.Knowledge{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "url"},
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
},
{
name: "共享只读", ids: []string{"a"}, status: 403,
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 8},
share: &downloadShareStub{permission: types.OrgRoleViewer},
},
{
name: "密钥无此库权限", ids: []string{"a"}, status: 403,
kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7},
scope: &types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"other"}},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
svc := &downloadKnowledgeStub{}
if tc.item != nil {
svc.items = []*types.Knowledge{tc.item}
}
w := runBatchDownload(t, svc, tc.ids, tc.kb, tc.share, tc.scope)
require.Equal(t, tc.status, w.Code, w.Body.String())
require.Empty(t, svc.opened)
require.NotContains(t, w.Header().Get("Content-Type"), "zip")
})
}
}
func TestBatchDownloadKnowledgeFailureDoesNotReturnPartialZIP(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
svc := &downloadKnowledgeStub{
items: []*types.Knowledge{
{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "a"},
{ID: "b", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "b"},
},
names: map[string]string{"a": "first.txt"}, failID: "b",
}
w := runBatchDownload(t, svc, []string{"a", "b"},
&types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil)
require.Equal(t, http.StatusInternalServerError, w.Code)
require.Contains(t, w.Header().Get("Content-Type"), "json")
require.Empty(t, w.Header().Get("Content-Disposition"))
entries, err := os.ReadDir(os.TempDir())
require.NoError(t, err)
require.Empty(t, entries)
}
type downloadTrackingReader struct {
io.Reader
closed bool
}
func (r *downloadTrackingReader) Close() error {
r.closed = true
return nil
}
func TestBatchDownloadKnowledgeSkipsEntriesWithoutOriginalFiles(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
svc := &downloadKnowledgeStub{
items: []*types.Knowledge{
{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-a"},
{ID: "url", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "url"},
},
names: map[string]string{"a": "keep.txt"},
}
w := runBatchDownload(t, svc, []string{"a", "url"},
&types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil)
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
reader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
require.NoError(t, err)
require.Len(t, reader.File, 1)
require.Equal(t, "keep.txt", reader.File[0].Name)
require.Equal(t, []string{"a"}, svc.opened)
}
func TestBatchDownloadKnowledgeAllowsSharedEditor(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
svc := &downloadKnowledgeStub{
items: []*types.Knowledge{{ID: "a", TenantID: 8, KnowledgeBaseID: "kb-1", FilePath: "stored-a"}},
names: map[string]string{"a": "shared.txt"},
expectedTenant: 8,
}
w := runBatchDownload(t, svc, []string{"a"},
&types.KnowledgeBase{ID: "kb-1", TenantID: 8},
&downloadShareStub{permission: types.OrgRoleEditor}, nil)
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
require.Equal(t, []string{"a"}, svc.opened)
}
func TestBatchDownloadKnowledgeRejectsWhenBusy(t *testing.T) {
for i := 0; i < maxConcurrentBatchDownloads; i++ {
batchDownloadSlots <- struct{}{}
}
t.Cleanup(func() {
for i := 0; i < maxConcurrentBatchDownloads; i++ {
<-batchDownloadSlots
}
})
svc := &downloadKnowledgeStub{
items: []*types.Knowledge{{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "a"}},
names: map[string]string{"a": "a.txt"},
}
w := runBatchDownload(t, svc, []string{"a"},
&types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil)
require.Equal(t, http.StatusTooManyRequests, w.Code, w.Body.String())
require.Empty(t, svc.opened)
}
func TestKnowledgeDownloadArchiveEnforcesActualByteLimitAndClosesFiles(t *testing.T) {
for _, limit := range []int64{4, 5} {
var output bytes.Buffer
file := &downloadTrackingReader{Reader: strings.NewReader("12345")}
err := writeKnowledgeDownloadArchive(context.Background(), &output,
[]knowledgeDownloadEntry{{ID: "a"}},
func(context.Context, string) (io.ReadCloser, string, error) { return file, "a.txt", nil }, limit)
if limit == 4 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.True(t, file.closed)
}
}
func TestKnowledgeDownloadArchiveStopsWhenCancelled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
var output bytes.Buffer
err := writeKnowledgeDownloadArchive(ctx, &output, []knowledgeDownloadEntry{{ID: "a"}},
func(context.Context, string) (io.ReadCloser, string, error) {
t.Fatal("取消后不应继续读取文件")
return nil, "", nil
}, 100)
require.ErrorIs(t, err, context.Canceled)
require.Zero(t, output.Len())
}
func TestKnowledgeDownloadArchiveCancelsAfterOpenAndClosesFile(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
file := &downloadTrackingReader{Reader: strings.NewReader("12345")}
err := writeKnowledgeDownloadArchive(ctx, io.Discard, []knowledgeDownloadEntry{{ID: "a"}},
func(context.Context, string) (io.ReadCloser, string, error) {
cancel()
return file, "a.txt", nil
}, 100)
require.True(t, isKnowledgeDownloadCanceled(err), err)
require.True(t, file.closed)
requireAppHTTP(t, mapKnowledgeDownloadError(err), http.StatusBadRequest)
}
func TestMapKnowledgeDownloadError(t *testing.T) {
requireAppHTTP(t, mapKnowledgeDownloadError(context.Canceled), http.StatusBadRequest)
requireAppHTTP(t, mapKnowledgeDownloadError(context.DeadlineExceeded), http.StatusBadRequest)
}
func requireAppHTTP(t *testing.T, err error, status int) {
t.Helper()
appErr, ok := apperrors.IsAppError(err)
require.True(t, ok, err)
require.Equal(t, status, appErr.HTTPCode)
}
func TestKnowledgeDownloadNamesAreSafeForWindowsAndUTF8(t *testing.T) {
used := map[string]bool{}
for _, name := range []string{
"../报告.txt", `C:\secret\Report.TXT`, "report.txt", "CON.txt", "COM¹.txt", "a:b?.txt", "..",
strings.Repeat("中", 100) + ".pdf",
} {
got := uniqueKnowledgeDownloadName(name, used)
require.True(t, utf8.ValidString(got))
require.NotContains(t, got, "/")
require.NotContains(t, got, "\\")
require.NotContains(t, got, ":")
require.NotEqual(t, "CON.txt", got)
require.NotEqual(t, "COM¹.txt", got)
require.Less(t, len(got), 255)
}
require.True(t, used["report (2).txt"])
}
func TestKnowledgeDownloadZipPathKeepsSafeFolders(t *testing.T) {
used := map[string]bool{}
require.Equal(t, "docs/spec/a.md", uniqueKnowledgeDownloadZipPath(`..\docs/spec`, "a.md", used))
require.Equal(t, "docs/spec/a (2).md", uniqueKnowledgeDownloadZipPath("docs/spec", "../a.md", used))
got := uniqueKnowledgeDownloadZipPath("/abs/../secret", "CON.txt", map[string]bool{})
require.Equal(t, "abs/secret/_CON.txt", got)
require.NotContains(t, got, "..")
require.False(t, strings.HasPrefix(got, "/"))
}
func TestBatchDownloadKnowledgePreservesFolderPaths(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
svc := &downloadKnowledgeStub{
items: []*types.Knowledge{{
ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-a", FolderPath: "docs/spec",
}},
names: map[string]string{"a": "design.md"},
}
w := runBatchDownload(t, svc, []string{"a"},
&types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil)
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
reader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
require.NoError(t, err)
require.Len(t, reader.File, 1)
require.Equal(t, "docs/spec/design.md", reader.File[0].Name)
}

View File

@ -307,6 +307,7 @@ func TestKnowledgeReadRoutesDeclareRetrieveCapability(t *testing.T) {
{http.MethodGet, "/api/v1/knowledge-bases/:id"},
{http.MethodPost, "/api/v1/knowledge-bases/:id/hybrid-search"},
{http.MethodGet, "/api/v1/knowledge-bases/:id/knowledge"},
{http.MethodPost, "/api/v1/knowledge-bases/:id/knowledge/batch-download"},
{http.MethodGet, "/api/v1/knowledge/:id"},
{http.MethodGet, "/api/v1/knowledge/:id/download"},
{http.MethodPost, "/api/v1/knowledge-bases/:id/faq/search"},

View File

@ -1,6 +1,7 @@
package router
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
@ -108,3 +109,45 @@ func TestKnowledgeDownloadRejectsReadOnlySharedKB(t *testing.T) {
require.Equal(t, http.StatusForbidden, rec.Code, "body=%s", rec.Body.String())
}
func TestBatchKnowledgeDownloadRejectsTenantViewer(t *testing.T) {
engine := newKnowledgeDownloadRouteTestEngine(
t,
types.TenantRoleViewer,
&types.Knowledge{ID: "knowledge-own", KnowledgeBaseID: "kb-own", TenantID: 1},
&types.KnowledgeBase{ID: "kb-own", TenantID: 1},
nil,
)
rec := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodPost,
"/api/v1/knowledge-bases/kb-own/knowledge/batch-download",
bytes.NewBufferString(`{"ids":["knowledge-own"]}`),
)
req.Header.Set("Content-Type", "application/json")
engine.ServeHTTP(rec, req)
require.Equal(t, http.StatusForbidden, rec.Code, "body=%s", rec.Body.String())
}
func TestBatchKnowledgeDownloadRejectsReadOnlySharedKB(t *testing.T) {
engine := newKnowledgeDownloadRouteTestEngine(
t,
types.TenantRoleContributor,
&types.Knowledge{ID: "knowledge-shared", KnowledgeBaseID: "kb-shared", TenantID: 2},
&types.KnowledgeBase{ID: "kb-shared", TenantID: 2},
&downloadKBShareStub{permission: types.OrgRoleViewer, source: 2},
)
rec := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodPost,
"/api/v1/knowledge-bases/kb-shared/knowledge/batch-download",
bytes.NewBufferString(`{"ids":["knowledge-shared"]}`),
)
req.Header.Set("Content-Type", "application/json")
engine.ServeHTTP(rec, req)
require.Equal(t, http.StatusForbidden, rec.Code, "body=%s", rec.Body.String())
}

View File

@ -74,6 +74,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl
kb.POST("/url", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateKnowledgeFromURL)
kb.POST("/manual", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateManualKnowledge)
kbRead.GET("", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledge)
// 原始文件下载沿用单文件下载的 Contributor + Editor 权限边界。
kbRead.POST("/batch-download", g.Contributor(), g.KBAccessWrite("id"), handler.BatchDownloadKnowledge)
kbRead.GET("/folders", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledgeFolders)
kb.PUT("/folders", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.RenameKnowledgeFolder)
// Clearing all contents under a KB is a destructive op; gate