From a7d35452972fd51ae6813852f4bfa73e2d41a110 Mon Sep 17 00:00:00 2001 From: ttommybot Date: Wed, 16 Sep 2026 15:40:10 +0800 Subject: [PATCH] feat(knowledge): add batch file downloads (#3300) * feat(knowledge): add batch file downloads * fix(knowledge): harden batch file downloads Skip entries without original files, keep folder paths in the ZIP, map client cancel to 400, and cap concurrent archives so a full-select download is usable without 500s or disk pile-up. * fix(knowledge): format batch-download tests for pre-push lint --------- Co-authored-by: wizardchen --- client/README.md | 6 + client/README_EN.md | 6 + client/knowledge.go | 46 ++ docs/api/knowledge.md | 28 +- docs/docs.go | 101 +++++ docs/swagger.json | 101 +++++ docs/swagger.yaml | 65 +++ frontend/src/api/knowledge-base/index.ts | 9 + frontend/src/i18n/locales/en-US.ts | 9 + frontend/src/i18n/locales/ja-JP.ts | 9 + frontend/src/i18n/locales/ko-KR.ts | 9 + frontend/src/i18n/locales/ru-RU.ts | 9 + frontend/src/i18n/locales/zh-CN.ts | 9 + frontend/src/utils/request.ts | 13 + .../src/views/knowledge/KnowledgeBase.vue | 83 +++- .../components/DocumentActionMenu.vue | 2 +- .../knowledge/components/DocumentBatchBar.vue | 47 ++- .../components/DocumentBatchDownload.test.ts | 35 ++ .../knowledge/components/DocumentCardView.vue | 2 +- .../knowledge/components/DocumentListView.vue | 4 +- .../knowledgeDownloadFileName.test.ts | 10 +- .../knowledge/knowledgeDownloadFileName.ts | 6 + internal/handler/knowledge_download.go | 361 ++++++++++++++++ internal/handler/knowledge_download_test.go | 393 ++++++++++++++++++ .../router_api_key_capabilities_test.go | 1 + .../router/router_knowledge_download_test.go | 43 ++ internal/router/routes_knowledge.go | 2 + 27 files changed, 1387 insertions(+), 22 deletions(-) create mode 100644 frontend/src/views/knowledge/components/DocumentBatchDownload.test.ts create mode 100644 internal/handler/knowledge_download.go create mode 100644 internal/handler/knowledge_download_test.go diff --git a/client/README.md b/client/README.md index a0b7be99..60033141 100644 --- a/client/README.md +++ b/client/README.md @@ -91,6 +91,12 @@ knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), kb.ID, if err != nil { // 处理错误 } + +// 将同一知识库中的原始文件打包下载为 ZIP(最多 200 个 ID,合计 512 MiB) +err = apiClient.DownloadKnowledgeFiles(context.Background(), kb.ID, []string{knowledge.ID}, "knowledge-files.zip") +if err != nil { + // 处理错误 +} ``` ### 示例:创建会话并进行问答 diff --git a/client/README_EN.md b/client/README_EN.md index 7c75e154..1c3231c7 100644 --- a/client/README_EN.md +++ b/client/README_EN.md @@ -91,6 +91,12 @@ knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), kb.ID, if err != nil { // Handle error } + +// Download original files from one knowledge base as a ZIP (max 200 IDs, 512 MiB) +err = apiClient.DownloadKnowledgeFiles(context.Background(), kb.ID, []string{knowledge.ID}, "knowledge-files.zip") +if err != nil { + // Handle error +} ``` ### Example: Create Session and Chat diff --git a/client/knowledge.go b/client/knowledge.go index 1c6efe16..96ba754c 100644 --- a/client/knowledge.go +++ b/client/knowledge.go @@ -557,6 +557,52 @@ func (c *Client) OpenKnowledgeFile(ctx context.Context, knowledgeID string) (str return filename, resp.Body, nil } +// BatchDownloadKnowledgeRequest selects documents in one knowledge base for ZIP download. +type BatchDownloadKnowledgeRequest struct { + IDs []string `json:"ids"` +} + +// DownloadKnowledgeFiles downloads a ZIP of original files for the given +// knowledge IDs to destPath. On any error after the file is opened, the +// partial file is removed. +func (c *Client) DownloadKnowledgeFiles(ctx context.Context, knowledgeBaseID string, ids []string, destPath string) error { + _, body, err := c.OpenKnowledgeFilesArchive(ctx, knowledgeBaseID, ids) + if err != nil { + return err + } + defer body.Close() + + out, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + if _, err := io.Copy(out, body); err != nil { + _ = out.Close() + _ = os.Remove(destPath) + return fmt.Errorf("failed to copy response body: %w", err) + } + return out.Close() +} + +// OpenKnowledgeFilesArchive starts a batch download and returns the +// server-suggested ZIP filename and a streaming reader. Callers MUST Close +// the returned reader. The request uses the streaming HTTP client so the +// default 30s timeout does not cut off large archives. +func (c *Client) OpenKnowledgeFilesArchive(ctx context.Context, knowledgeBaseID string, ids []string) (string, io.ReadCloser, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/batch-download", knowledgeBaseID) + resp, err := c.doRequestStream(ctx, http.MethodPost, path, BatchDownloadKnowledgeRequest{IDs: ids}, nil) + if err != nil { + return "", nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return "", nil, newAPIError(resp.StatusCode, body) + } + filename := filenameFromContentDisposition(resp.Header.Get("Content-Disposition")) + return filename, resp.Body, nil +} + // filenameFromContentDisposition extracts the filename parameter from a // Content-Disposition header. Returns "" on any parse failure or missing // parameter — callers fall back to their own default in that case. diff --git a/docs/api/knowledge.md b/docs/api/knowledge.md index f0a091c8..ec095058 100644 --- a/docs/api/knowledge.md +++ b/docs/api/knowledge.md @@ -21,6 +21,7 @@ | POST | `/knowledge/:id/reparse` | 重新解析知识(异步) | | POST | `/knowledge/:id/cancel-parse` | 取消正在进行的解析任务 | | GET | `/knowledge/:id/download` | 下载原始文件(attachment) | +| POST | `/knowledge-bases/:id/knowledge/batch-download` | 将所选文档打包下载为 ZIP | | GET | `/knowledge/:id/preview` | 内联预览文件(按扩展名设置 Content-Type) | | PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 | | PUT | `/knowledge/tags` | 批量更新知识标签 | @@ -712,7 +713,32 @@ curl --location --request POST 'http://localhost:8080/api/v1/knowledge/4c4e7c1a- } ``` -## GET `/knowledge/:id/download` - 下载原始文件 +## 文件下载 + +### POST `/knowledge-bases/:id/knowledge/batch-download` - 批量下载 + +`POST /api/v1/knowledge-bases/:id/knowledge/batch-download`,请求体为 `{"ids":["文档ID1","文档ID2"]}`。 + +- 权限与单文件下载一致:JWT 用户需要 Contributor 身份及知识库 Editor 权限;API Key 需要 `retrieve` 能力或完整访问权限,并受知识库白名单限制。 +- 每次最多 200 个 ID(重复 ID 只下载一次),原始文件实际读取字节合计最多 512 MiB。超限请分批下载。服务端同时最多处理 4 个批量下载请求,过多时返回 429。 +- 成功返回 `application/zip` 附件;保留文件内容、中文文件名和知识库 `folder_path`。同名文件自动追加数字,移除不安全路径和 Windows 不支持的文件名字符。 +- 手工编写的文档导出为 Markdown。没有原始文件的网页条目会被跳过;若跳过后没有任何可下载文件,返回 400。FAQ 条目通常没有原始文件,不能通过此接口导出。 +- 任一条目无权访问、不属于当前知识库或读取失败时,整个请求失败,不返回残缺 ZIP。请求取消时返回 400,而不是 500。 +- 压缩包在服务器临时目录生成后发送,请求结束或失败后清理临时文件;不包含分块、向量、标签、权限、Wiki 或图谱。 + +页面操作:进入知识库的文档页 → 点击「批量管理」→ 勾选文件,或「全选已加载」→「批量下载」。全选只包含列表已加载的文档;可先继续加载更多,再选择,单批仍受 200 项限制。没有原始文件的条目会自动跳过。ZIP 解压后可按文件夹批量上传到其他知识库。 + +**请求**: + +```curl +curl --location --request POST 'http://localhost:8080/api/v1/knowledge-bases/kb-id/knowledge/batch-download' \ +--header 'X-API-Key: sk-xxxxx' \ +--header 'Content-Type: application/json' \ +--data '{"ids":["文档ID1","文档ID2"]}' \ +--output knowledge-files.zip +``` + +### GET `/knowledge/:id/download` - 单文件下载 以 `attachment` 方式下载知识对应的原始文件。 diff --git a/docs/docs.go b/docs/docs.go index 25fb2605..913b994b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -5193,6 +5193,91 @@ const docTemplate = `{ } } }, + "/knowledge-bases/{id}/knowledge/batch-download": { + "post": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。", + "consumes": [ + "application/json" + ], + "produces": [ + "application/zip" + ], + "tags": [ + "知识管理" + ], + "summary": "批量下载知识文件", + "parameters": [ + { + "type": "string", + "description": "知识库ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "文档ID列表", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/internal_handler.BatchDownloadKnowledgeRequest" + } + } + ], + "responses": { + "200": { + "description": "ZIP 压缩包", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "429": { + "description": "Too Many Requests", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledge-bases/{id}/knowledge/file": { "post": { "security": [ @@ -23574,6 +23659,22 @@ const docTemplate = `{ } } }, + "internal_handler.BatchDownloadKnowledgeRequest": { + "type": "object", + "required": [ + "ids" + ], + "properties": { + "ids": { + "type": "array", + "maxItems": 200, + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, "internal_handler.CopyKnowledgeBaseRequest": { "type": "object", "required": [ diff --git a/docs/swagger.json b/docs/swagger.json index 1f1c54b2..4150f135 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5186,6 +5186,91 @@ } } }, + "/knowledge-bases/{id}/knowledge/batch-download": { + "post": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。", + "consumes": [ + "application/json" + ], + "produces": [ + "application/zip" + ], + "tags": [ + "知识管理" + ], + "summary": "批量下载知识文件", + "parameters": [ + { + "type": "string", + "description": "知识库ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "文档ID列表", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/internal_handler.BatchDownloadKnowledgeRequest" + } + } + ], + "responses": { + "200": { + "description": "ZIP 压缩包", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "429": { + "description": "Too Many Requests", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledge-bases/{id}/knowledge/file": { "post": { "security": [ @@ -23567,6 +23652,22 @@ } } }, + "internal_handler.BatchDownloadKnowledgeRequest": { + "type": "object", + "required": [ + "ids" + ], + "properties": { + "ids": { + "type": "array", + "maxItems": 200, + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, "internal_handler.CopyKnowledgeBaseRequest": { "type": "object", "required": [ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e47d0677..c654366d 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -5030,6 +5030,17 @@ definitions: - ids - kb_id type: object + internal_handler.BatchDownloadKnowledgeRequest: + properties: + ids: + items: + type: string + maxItems: 200 + minItems: 1 + type: array + required: + - ids + type: object internal_handler.CopyKnowledgeBaseRequest: properties: source_id: @@ -9720,6 +9731,60 @@ paths: summary: 获取知识列表 tags: - 知识管理 + /knowledge-bases/{id}/knowledge/batch-download: + post: + consumes: + - application/json + description: 将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。 + parameters: + - description: 知识库ID + in: path + name: id + required: true + type: string + - description: 文档ID列表 + in: body + name: request + required: true + schema: + $ref: '#/definitions/internal_handler.BatchDownloadKnowledgeRequest' + produces: + - application/zip + responses: + "200": + description: ZIP 压缩包 + schema: + type: file + "400": + description: Bad Request + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "403": + description: Forbidden + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "429": + description: Too Many Requests + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + security: + - Bearer: [] + - ApiKeyAuth: [] + summary: 批量下载知识文件 + tags: + - 知识管理 /knowledge-bases/{id}/knowledge/file: post: consumes: diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 791e0da9..7fe7f510 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -378,6 +378,15 @@ export function downKnowledgeDetails(id: string) { return getDown(`/api/v1/knowledge/${id}/download`); } +// 使用已有登录和租户请求头下载 ZIP,不将凭据放入下载链接。 +export function batchDownloadKnowledge(kbId: string, ids: string[], signal?: AbortSignal): Promise { + return post(`/api/v1/knowledge-bases/${encodeURIComponent(kbId)}/knowledge/batch-download`, { ids }, { + responseType: 'blob', + timeout: 300000, + signal, + }); +} + export function previewKnowledgeFile(id: string) { return getDown(`/api/v1/knowledge/${id}/preview`); } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index c90d8a50..50d49122 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -710,6 +710,15 @@ export default { selectedCount: '{count} selected', clearSelection: 'Deselect all', batchDelete: 'Delete selected', + batchDownload: 'Download selected', + batchDownloading: 'Preparing download…', + batchDownloadHint: 'Download a ZIP of up to 200 documents and 512 MiB of original content per batch. Select all includes loaded documents only; web pages without original files are skipped. The ZIP keeps knowledge-base folders.', + batchDownloadStarted: 'ZIP saving started. Extract it, then upload the files and folders.', + batchDownloadFailed: 'Batch download failed. Please try again.', + batchDownloadSkipped: 'Skipped {count} documents that have no original file.', + batchDownloadNoFiles: 'None of the selected documents have an original file to download.', + batchDownloadTooLarge: 'Selected files total more than 512 MiB. Choose fewer documents and try again.', + selectLoaded: 'Select loaded', confirmBatchDeleteDocument: 'Delete {count} selected documents? This action cannot be undone.', deleteSubmitted: 'Delete request submitted. Waiting for completion.', deletePending: 'Deletion is still pending. Refresh later to check the result.', diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts index c0784eed..6f04a6ca 100644 --- a/frontend/src/i18n/locales/ja-JP.ts +++ b/frontend/src/i18n/locales/ja-JP.ts @@ -710,6 +710,15 @@ export default { selectedCount: '{count}件選択中', clearSelection: 'すべて選択解除', batchDelete: '選択項目を削除', + batchDownload: '選択項目をダウンロード', + batchDownloading: 'ダウンロードを準備中…', + batchDownloadHint: '1回につき最大200件、元ファイル合計512 MiBまでZIPでダウンロードできます。全選択は読み込み済み文書のみ対象で、元ファイルのないウェブページはスキップされます。ZIPはフォルダ構造を保持します。', + batchDownloadStarted: 'ZIPの保存を開始しました。展開してからファイルとフォルダをアップロードしてください。', + batchDownloadFailed: '一括ダウンロードに失敗しました。もう一度お試しください。', + batchDownloadSkipped: '元ファイルのない文書を{count}件スキップしました。', + batchDownloadNoFiles: '選択した文書にダウンロードできる元ファイルがありません。', + batchDownloadTooLarge: '選択したファイルの合計が512 MiBを超えています。件数を減らして分批ダウンロードしてください。', + selectLoaded: '読み込み済みを選択', confirmBatchDeleteDocument: '選択した{count}件のドキュメントを削除しますか?この操作は取り消せません。', deleteSubmitted: '削除リクエストを送信しました。完了までお待ちください。', deletePending: '削除はまだ処理中です。後で更新して結果を確認してください。', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 4199aa45..09e593b6 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -6736,6 +6736,15 @@ export default { selectedCount: '{count}개 선택됨', clearSelection: '선택 해제', batchDelete: '선택 삭제', + batchDownload: '일괄 다운로드', + batchDownloading: '다운로드 준비 중…', + batchDownloadHint: '한 번에 최대 200개 문서, 원본 합계 512 MiB를 ZIP으로 다운로드합니다. 전체 선택은 불러온 문서만 포함하며, 원본이 없는 웹 페이지는 건너뜁니다. ZIP은 폴더 구조를 유지합니다.', + batchDownloadStarted: 'ZIP 저장을 시작했습니다. 압축을 푼 후 파일과 폴더를 업로드하세요.', + batchDownloadFailed: '일괄 다운로드에 실패했습니다. 다시 시도하세요.', + batchDownloadSkipped: '원본 파일이 없는 문서 {count}개를 건너뛰었습니다.', + batchDownloadNoFiles: '선택한 문서에 다운로드할 원본 파일이 없습니다.', + batchDownloadTooLarge: '선택한 파일 합계가 512 MiB를 초과합니다. 항목을 줄인 뒤 나눠 다운로드하세요.', + selectLoaded: '불러온 문서 선택', confirmBatchDeleteDocument: '선택한 {count}개 문서를 삭제하시겠습니까? 삭제 후 복구할 수 없습니다.', deleteSubmitted: '삭제 요청이 제출되었습니다. 완료를 기다리는 중입니다.', deletePending: '삭제가 아직 완료되지 않았습니다. 잠시 후 새로고침하여 결과를 확인하세요.', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index e7256934..60ec49ae 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -6736,6 +6736,15 @@ export default { selectedCount: 'Выбрано: {count}', clearSelection: 'Снять выделение', batchDelete: 'Удалить выбранные', + batchDownload: 'Скачать выбранные', + batchDownloading: 'Подготовка архива…', + batchDownloadHint: 'ZIP: до 200 документов и 512 МиБ исходных файлов за раз. Выбор всех включает только загруженные документы; веб-страницы без исходных файлов пропускаются. ZIP сохраняет папки базы знаний.', + batchDownloadStarted: 'Началось сохранение ZIP. Распакуйте архив, затем загрузите файлы и папки.', + batchDownloadFailed: 'Не удалось скачать файлы. Повторите попытку.', + batchDownloadSkipped: 'Пропущено документов без исходного файла: {count}.', + batchDownloadNoFiles: 'У выбранных документов нет исходных файлов для скачивания.', + batchDownloadTooLarge: 'Суммарный размер выбранных файлов превышает 512 МиБ. Уменьшите выбор и скачайте частями.', + selectLoaded: 'Выбрать загруженные', confirmBatchDeleteDocument: 'Удалить {count} выбранных документов? Это действие нельзя отменить.', deleteSubmitted: 'Запрос на удаление отправлен. Ожидание завершения.', deletePending: 'Удаление ещё не завершено. Обновите страницу позже, чтобы проверить результат.', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 22d560e4..518c6b6d 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -6738,6 +6738,15 @@ export default { selectedCount: '已选 {count} 项', clearSelection: '取消选择', batchDelete: '批量删除', + batchDownload: '批量下载', + batchDownloading: '正在打包下载…', + batchDownloadHint: '下载所选文件为 ZIP,每批最多 200 项、原始文件合计 512 MiB。全选仅包含已加载的文档;没有原始文件的网页条目会自动跳过。ZIP 会保留知识库文件夹。', + batchDownloadStarted: '已开始保存 ZIP,解压后即可按文件夹批量上传文件', + batchDownloadFailed: '批量下载失败,请稍后重试', + batchDownloadSkipped: '已跳过 {count} 个没有原始文件的文档', + batchDownloadNoFiles: '所选文档没有可下载的原始文件', + batchDownloadTooLarge: '所选文件合计超过 512 MiB,请减少选择后分批下载', + selectLoaded: '全选已加载', confirmBatchDeleteDocument: '确认删除选中的 {count} 个文档?删除后将无法恢复。', deleteSubmitted: '删除任务已提交,正在等待完成。', deletePending: '删除尚未完成,请稍后刷新查看结果。', diff --git a/frontend/src/utils/request.ts b/frontend/src/utils/request.ts index 80660245..f07ce8e1 100644 --- a/frontend/src/utils/request.ts +++ b/frontend/src/utils/request.ts @@ -137,6 +137,19 @@ instance.interceptors.response.use( if (!error.response) { return Promise.reject({ message: t('error.networkError') }); } + + // 文件下载失败时服务端仍返回 JSON;先还原错误信息,避免被 Blob 隐藏。 + // 不依赖 Content-Type:网关可能把错误改成 text/plain 或空类型。 + if (typeof Blob !== 'undefined' && error.response.data instanceof Blob) { + try { + const text = (await error.response.data.text()).trim(); + if (text.startsWith('{') || text.startsWith('[') || error.response.data.type.includes('json')) { + error.response.data = JSON.parse(text); + } + } catch { + // 非法 JSON 继续使用原有错误处理。 + } + } // 公开接口(auto-setup / login / register / oidc)的 401 不走 refresh 逻辑,直接返回错误 if ((error.response.status === 401 || error.response.status === 403) && isPublicAuthRequest(originalRequest?.url)) { diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index d9d3c1c8..5bf2535c 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -32,6 +32,7 @@ import { reparseKnowledge, cancelKnowledgeParse, batchDeleteKnowledge, + batchDownloadKnowledge, delKnowledgeDetails, batchReparseKnowledge, getKnowledgeSpans, @@ -42,6 +43,7 @@ import { downKnowledgeDetails, type KnowledgeFolderTree, } from "@/api/knowledge-base/index"; +import { isBatchDownloadableKnowledge } from './knowledgeDownloadFileName'; import { waitForKnowledgeDeletion } from '@/utils/knowledgeDeletion'; import { knowledgeSpansPayloadHasTrace } from '@/utils/knowledgeTrace'; import FAQEntryManager from './components/FAQEntryManager.vue'; @@ -441,6 +443,67 @@ let lastSelectedIndex = -1; const batchDeleting = ref(false); const batchReparsing = ref(false); const batchTagging = ref(false); +const batchDownloading = ref(false); +let batchDownloadController: AbortController | undefined; + +const MAX_BATCH_DOWNLOAD_FILES = 200; +const MAX_BATCH_DOWNLOAD_BYTES = 512 * 1024 * 1024; + +const handleBatchDownload = async () => { + if (batchDownloading.value || batchDeleting.value || batchReparsing.value || batchTagging.value || selectedIds.value.size === 0) return; + const selected = Array.from(selectedIds.value); + const itemsById = new Map((cardList.value || []).map((item) => [item.id, item])); + const ids = selected.filter((id) => isBatchDownloadableKnowledge(itemsById.get(id))); + const skipped = selected.length - ids.length; + if (ids.length === 0) { + MessagePlugin.warning(t('knowledgeBase.batchDownloadNoFiles')); + return; + } + if (ids.length > MAX_BATCH_DOWNLOAD_FILES) { + MessagePlugin.warning(t('knowledgeBase.batchDownloadHint')); + return; + } + let knownBytes = 0; + for (const id of ids) { + const size = Number(itemsById.get(id)?.file_size); + if (Number.isFinite(size) && size > 0) knownBytes += size; + } + if (knownBytes > MAX_BATCH_DOWNLOAD_BYTES) { + MessagePlugin.warning(t('knowledgeBase.batchDownloadTooLarge')); + return; + } + if (skipped > 0) { + MessagePlugin.warning(t('knowledgeBase.batchDownloadSkipped', { count: skipped })); + } + const controller = new AbortController(); + batchDownloadController = controller; + batchDownloading.value = true; + try { + const blob = await batchDownloadKnowledge(kbId.value, ids, controller.signal); + if (controller.signal.aborted) return; + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = `knowledge-files-${Date.now()}.zip`; + document.body.appendChild(link); + link.click(); + link.remove(); + // 延迟释放,给浏览器留出开始保存文件的时间。 + window.setTimeout(() => URL.revokeObjectURL(url), 60000); + MessagePlugin.success(t('knowledgeBase.batchDownloadStarted')); + } catch (error: any) { + if (!controller.signal.aborted) { + MessagePlugin.error(error?.message || t('knowledgeBase.batchDownloadFailed')); + } + } finally { + if (batchDownloadController === controller) { + batchDownloading.value = false; + batchDownloadController = undefined; + } + } +}; +watch(kbId, () => batchDownloadController?.abort()); +onUnmounted(() => batchDownloadController?.abort()); const batchTagDialogVisible = ref(false); const batchTagPreSelectedIds = computed(() => { const ids = Array.from(selectedIds.value); @@ -497,7 +560,7 @@ const awaitBatchReparseReflection = async (ids: string[]) => { }; const confirmBatchReparse = async () => { - if (batchReparsing.value || batchDeleting.value || selectedIds.value.size === 0) return; + if (batchReparsing.value || batchDeleting.value || batchDownloading.value || selectedIds.value.size === 0) return; const allIds = Array.from(selectedIds.value); const ids = allIds.filter((id) => { const item = cardList.value.find((c) => c.id === id); @@ -2062,7 +2125,7 @@ const { itemSelector: '.knowledge-card[data-select-id], .doc-list-row[data-select-id]', selectedIds, getItemId: (el) => el.dataset.selectId || null, - enabled: computed(() => canEdit.value && !isFAQ.value && cardList.value.length > 0), + enabled: computed(() => (canEdit.value || canDownloadKnowledge.value) && !isFAQ.value && cardList.value.length > 0), onSelectionStart: () => { batchMode.value = true; }, @@ -2096,7 +2159,7 @@ const deleteKnowledgeDocuments = async ( submit: () => Promise, batch: boolean, ) => { - if (batchDeleting.value || batchReparsing.value || ids.length === 0) return; + if (batchDeleting.value || batchReparsing.value || batchDownloading.value || ids.length === 0) return; const targetKbId = kbId.value; const generation = ++deleteGeneration; const isActive = () => generation === deleteGeneration && isCurrentKb(targetKbId); @@ -2163,7 +2226,7 @@ const confirmBatchDelete = () => { }; const handleBatchTag = () => { - if (batchDeleting.value || batchReparsing.value || batchTagging.value || selectedIds.value.size === 0) return; + if (batchDeleting.value || batchReparsing.value || batchTagging.value || batchDownloading.value || selectedIds.value.size === 0) return; batchTagDialogVisible.value = true; }; @@ -2594,6 +2657,12 @@ async function createNewSession(value: string): Promise {
+ + {{ $t(batchMode ? 'knowledgeBase.clearSelection' : 'menu.batchManage') }} +
diff --git a/frontend/src/views/knowledge/components/DocumentActionMenu.vue b/frontend/src/views/knowledge/components/DocumentActionMenu.vue index e9e115a3..2326f440 100644 --- a/frontend/src/views/knowledge/components/DocumentActionMenu.vue +++ b/frontend/src/views/knowledge/components/DocumentActionMenu.vue @@ -108,7 +108,7 @@ const fileName = computed(() => props.item.file_name || props.item.title || prop -
+
{{ $t('menu.batchManage') }}
diff --git a/frontend/src/views/knowledge/components/DocumentBatchBar.vue b/frontend/src/views/knowledge/components/DocumentBatchBar.vue index de5e7b6e..7a03515b 100644 --- a/frontend/src/views/knowledge/components/DocumentBatchBar.vue +++ b/frontend/src/views/knowledge/components/DocumentBatchBar.vue @@ -8,6 +8,9 @@ defineProps<{ deleteLoading?: boolean; reparseLoading?: boolean; tagLoading?: boolean; + downloadLoading?: boolean; + canDownload?: boolean; + canMutate?: boolean; // When true the bar stays visible even with 0 selections, so users can exit // batch mode from here without selecting anything first. visible?: boolean; @@ -22,6 +25,8 @@ const emit = defineEmits<{ (e: 'reparse'): void; (e: 'batchTag'): void; (e: 'moveToFolder', folderPath: string): void; + (e: 'download'): void; + (e: 'selectLoaded'): void; }>(); const { t } = useI18n(); @@ -36,32 +41,48 @@ const folderPickerVisible = ref(false);
{{ t('knowledgeBase.selectedCount', { count }) }} - + + {{ t('knowledgeBase.selectLoaded') }} + + {{ t('knowledgeBase.clearSelection') }}
- + + + + {{ t(downloadLoading ? 'knowledgeBase.batchDownloading' : 'knowledgeBase.batchDownload') }} + + + + + + :disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading" :loading="reparseLoading" @click.stop> {{ t('knowledgeBase.rebuildDocument') }} - {{ t('knowledgeBase.batchTag') }} - + :disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading"> {{ t('knowledgeBase.moveToFolder.action') }} @@ -73,11 +94,11 @@ const folderPickerVisible = ref(false); - + :disabled="count === 0 || deleteLoading || reparseLoading || tagLoading || downloadLoading" :loading="deleteLoading" @click.stop> {{ t('knowledgeBase.batchDelete') }} @@ -93,7 +114,7 @@ const folderPickerVisible = ref(false); position: relative; z-index: 5; width: 100%; - max-width: 560px; + max-width: 920px; margin: 0 auto; padding: 0 4px; box-sizing: border-box; @@ -101,6 +122,7 @@ const folderPickerVisible = ref(false); .batch-bar-inner { display: flex; + flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px; @@ -113,6 +135,7 @@ const folderPickerVisible = ref(false); .batch-bar-left { display: flex; + flex-wrap: wrap; align-items: center; gap: 4px; min-width: 0; @@ -147,6 +170,10 @@ const folderPickerVisible = ref(false); gap: 8px; } +.batch-download-trigger { + display: inline-flex; +} + .batch-bar-fade-enter-active, .batch-bar-fade-leave-active { transition: transform 0.2s ease, opacity 0.2s ease; diff --git a/frontend/src/views/knowledge/components/DocumentBatchDownload.test.ts b/frontend/src/views/knowledge/components/DocumentBatchDownload.test.ts new file mode 100644 index 00000000..fe13a87b --- /dev/null +++ b/frontend/src/views/knowledge/components/DocumentBatchDownload.test.ts @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { readFileSync } from 'node:fs' + +const knowledgeBase = readFileSync(new URL('../KnowledgeBase.vue', import.meta.url), 'utf8') +const batchBar = readFileSync(new URL('./DocumentBatchBar.vue', import.meta.url), 'utf8') +const listView = readFileSync(new URL('./DocumentListView.vue', import.meta.url), 'utf8') +const knowledgeApi = readFileSync(new URL('../../../api/knowledge-base/index.ts', import.meta.url), 'utf8') +const request = readFileSync(new URL('../../../utils/request.ts', import.meta.url), 'utf8') + +test('批量下载使用受认证的 Blob 请求并支持取消', () => { + assert.match(knowledgeApi, /knowledge-bases\/\$\{encodeURIComponent\(kbId\)\}\/knowledge\/batch-download/) + assert.match(knowledgeApi, /responseType:\s*'blob'/) + assert.match(knowledgeApi, /signal,/) + assert.match(knowledgeBase, /new AbortController\(\)/) + assert.match(knowledgeBase, /batchDownloadController\?\.abort\(\)/) +}) + +test('批量下载界面限制单批 200 项并保留只读权限边界', () => { + assert.match(batchBar, /v-if="canDownload"/) + assert.match(batchBar, /count > 200/) + assert.match(batchBar, /batch-download-trigger/) + assert.match(batchBar, /v-if="canMutate"/) + assert.match(knowledgeBase, /isBatchDownloadableKnowledge/) + assert.match(knowledgeBase, /ids\.length > MAX_BATCH_DOWNLOAD_FILES/) + assert.match(knowledgeBase, /:can-download="canDownloadKnowledge"/) + assert.match(knowledgeBase, /@select-loaded="toggleSelectAll\(true\)"/) + assert.match(listView, /v-if="canEdit \|\| canDownload"/) +}) + +test('Blob 格式的 JSON 错误会还原为可读消息', () => { + assert.match(request, /error\.response\.data instanceof Blob/) + assert.match(request, /JSON\.parse\(text\)/) + assert.match(request, /text\.startsWith\('\{'\)/) +}) diff --git a/frontend/src/views/knowledge/components/DocumentCardView.vue b/frontend/src/views/knowledge/components/DocumentCardView.vue index ec55b077..4484a521 100644 --- a/frontend/src/views/knowledge/components/DocumentCardView.vue +++ b/frontend/src/views/knowledge/components/DocumentCardView.vue @@ -348,7 +348,7 @@ const handleAction = (action: 'download' | 'edit' | 'view-trace' | 'reparse' | ' >
-
+
-
{{ t('knowledgeBase.columnName') }}
@@ -306,7 +306,7 @@ const handleAction = (action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | :class="{ selected: selectedIds.has(item.id), 'menu-open': moreOpen === item.id }" :data-select-id="item.id" role="row" @click="emit('open', item)">
-
diff --git a/frontend/src/views/knowledge/knowledgeDownloadFileName.test.ts b/frontend/src/views/knowledge/knowledgeDownloadFileName.test.ts index 18dee61d..3db49140 100644 --- a/frontend/src/views/knowledge/knowledgeDownloadFileName.test.ts +++ b/frontend/src/views/knowledge/knowledgeDownloadFileName.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { resolveKnowledgeDownloadFileName } from './knowledgeDownloadFileName.ts'; +import { isBatchDownloadableKnowledge, resolveKnowledgeDownloadFileName } from './knowledgeDownloadFileName.ts'; test('prefers the original filename over the extensionless display name', () => { assert.equal(resolveKnowledgeDownloadFileName({ @@ -20,6 +20,14 @@ test('falls back to the backend filename when no original filename is present', }), 'notes.txt'); }); +test('batch download skips web pages without an original file', () => { + assert.equal(isBatchDownloadableKnowledge({ type: 'manual' }), true); + assert.equal(isBatchDownloadableKnowledge({ type: 'file', file_path: 'stored' }), true); + assert.equal(isBatchDownloadableKnowledge({ type: 'url' }), false); + assert.equal(isBatchDownloadableKnowledge({ type: 'url', file_path: 'snapshot.html' }), true); + assert.equal(isBatchDownloadableKnowledge(undefined), false); +}); + test('adds the markdown extension to manual documents exactly once', () => { assert.equal(resolveKnowledgeDownloadFileName({ id: 'knowledge-3', diff --git a/frontend/src/views/knowledge/knowledgeDownloadFileName.ts b/frontend/src/views/knowledge/knowledgeDownloadFileName.ts index 08ceb8ce..74e991ef 100644 --- a/frontend/src/views/knowledge/knowledgeDownloadFileName.ts +++ b/frontend/src/views/knowledge/knowledgeDownloadFileName.ts @@ -13,3 +13,9 @@ export function resolveKnowledgeDownloadFileName(item: KnowledgeDownloadItem): s } return baseName; } + +export function isBatchDownloadableKnowledge(item?: { type?: string; file_path?: string } | null): boolean { + if (!item) return false; + if (item.type === 'manual') return true; + return Boolean(item.file_path); +} diff --git a/internal/handler/knowledge_download.go b/internal/handler/knowledge_download.go new file mode 100644 index 00000000..cf9ad0cd --- /dev/null +++ b/internal/handler/knowledge_download.go @@ -0,0 +1,361 @@ +package handler + +import ( + "archive/zip" + "context" + stderrors "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "time" + "unicode" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/filetransport" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/gin-gonic/gin" +) + +const ( + maxBatchDownloadFiles = 200 + maxBatchDownloadBytes int64 = 512 * 1024 * 1024 + maxConcurrentBatchDownloads = 4 +) + +var batchDownloadSlots = make(chan struct{}, maxConcurrentBatchDownloads) + +// BatchDownloadKnowledgeRequest 指定同一知识库中需要下载的文档。 +type BatchDownloadKnowledgeRequest struct { + IDs []string `json:"ids" binding:"required,min=1,max=200,dive,required,max=128"` +} + +type knowledgeDownloadEntry struct { + ID string + FolderPath string +} + +// BatchDownloadKnowledge godoc +// @Summary 批量下载知识文件 +// @Description 将同一知识库的最多 200 个文档打包为 ZIP,原始内容合计不超过 512 MiB。无原文件的条目会被跳过;无权访问、跨库或读取失败时不返回残缺压缩包。 +// @Tags 知识管理 +// @Accept json +// @Produce application/zip +// @Param id path string true "知识库ID" +// @Param request body BatchDownloadKnowledgeRequest true "文档ID列表" +// @Success 200 {file} file "ZIP 压缩包" +// @Failure 400 {object} errors.AppError +// @Failure 401 {object} errors.AppError +// @Failure 403 {object} errors.AppError +// @Failure 404 {object} errors.AppError +// @Failure 429 {object} errors.AppError +// @Failure 500 {object} errors.AppError +// @Security Bearer +// @Security ApiKeyAuth +// @Router /knowledge-bases/{id}/knowledge/batch-download [post] +func (h *KnowledgeHandler) BatchDownloadKnowledge(c *gin.Context) { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 64*1024) + var req BatchDownloadKnowledgeRequest + if err := c.ShouldBindJSON(&req); err != nil { + _ = c.Error(errors.NewBadRequestError("请选择 1~200 个文档进行批量下载")) + return + } + + kbID := c.Param("id") + grant, err := resolveHandlerKBAccessFor( + c, kbID, h.kbService, h.kbShareService, h.agentShareService, types.OrgRoleEditor, + ) + if err != nil { + _ = c.Error(err) + return + } + tenantID := grant.EffectiveTenantID + ctx := types.WithExecutionTenant(c.Request.Context(), tenantID) + ids := uniqueKnowledgeDownloadIDs(req.IDs) + if len(ids) == 0 { + _ = c.Error(errors.NewBadRequestError("文档 ID 不能为空")) + return + } + + items, err := h.kgService.GetKnowledgeBatch(ctx, tenantID, ids) + if err != nil { + _ = c.Error(errors.NewInternalServerError("无法获取待下载文档,请稍后重试")) + return + } + byID := make(map[string]*types.Knowledge, len(items)) + for _, item := range items { + if item != nil && item.KnowledgeBaseID == kbID && item.TenantID == tenantID { + byID[item.ID] = item + } + } + entries := make([]knowledgeDownloadEntry, 0, len(ids)) + // 在读取任何文件之前检查整批文档,防止混入其他知识库或租户的 ID。 + for _, id := range ids { + item := byID[id] + if item == nil { + _ = c.Error(errors.NewNotFoundError("部分文档不存在或不属于当前知识库,请刷新列表后重试")) + return + } + if !item.IsManual() && item.FilePath == "" { + continue + } + entries = append(entries, knowledgeDownloadEntry{ID: id, FolderPath: item.FolderPath}) + } + if len(entries) == 0 { + _ = c.Error(errors.NewBadRequestError("所选文档没有可下载的原始文件,请取消选择后重试")) + return + } + + if !tryAcquireBatchDownloadSlot() { + _ = c.Error(errors.NewTooManyRequestsError("当前批量下载过多,请稍后重试")) + return + } + defer releaseBatchDownloadSlot() + + // 先在临时文件中完整生成压缩包,避免读取失败时向用户返回残缺 ZIP。 + archive, err := os.CreateTemp("", "weknora-download-*.zip") + if err != nil { + _ = c.Error(errors.NewInternalServerError("无法创建下载压缩包,请稍后重试")) + return + } + archivePath := archive.Name() + defer func() { + _ = archive.Close() + _ = os.Remove(archivePath) + }() + if err := writeKnowledgeDownloadArchive( + ctx, archive, entries, h.kgService.GetKnowledgeFile, maxBatchDownloadBytes, + ); err != nil { + mapped := mapKnowledgeDownloadError(err) + if appErr, ok := errors.IsAppError(mapped); !ok || appErr.HTTPCode >= 500 { + logger.ErrorWithFields(ctx, err, nil) + } + _ = c.Error(mapped) + return + } + if _, err := archive.Seek(0, io.SeekStart); err != nil { + _ = c.Error(errors.NewInternalServerError("无法读取下载压缩包,请稍后重试")) + return + } + filename := "knowledge-files-" + time.Now().Format("20060102-150405") + ".zip" + if err := filetransport.Serve(c.Writer, c.Request, archive, filetransport.Options{ + Filename: filename, Download: true, ContentType: "application/zip", CacheControl: "private, no-store", + }); err != nil { + logger.Errorf(ctx, "Failed to send knowledge archive: %v", err) + } +} + +func tryAcquireBatchDownloadSlot() bool { + select { + case batchDownloadSlots <- struct{}{}: + return true + default: + return false + } +} + +func releaseBatchDownloadSlot() { + <-batchDownloadSlots +} + +func uniqueKnowledgeDownloadIDs(input []string) []string { + ids := make([]string, 0, len(input)) + seen := make(map[string]bool, len(input)) + for _, id := range input { + id = strings.TrimSpace(id) + if id != "" && !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + return ids +} + +func isKnowledgeDownloadCanceled(err error) bool { + return err != nil && (stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded)) +} + +func mapKnowledgeDownloadError(err error) error { + if err == nil { + return nil + } + if _, ok := errors.IsAppError(err); ok { + return err + } + if isKnowledgeDownloadCanceled(err) { + return errors.NewBadRequestError("下载已取消") + } + return err +} + +type knowledgeDownloadOpener func(context.Context, string) (io.ReadCloser, string, error) + +func writeKnowledgeDownloadArchive( + ctx context.Context, + output io.Writer, + entries []knowledgeDownloadEntry, + open knowledgeDownloadOpener, + limit int64, +) error { + writer := zip.NewWriter(output) + usedNames := make(map[string]bool, len(entries)) + var total int64 + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return err + } + file, filename, err := open(ctx, entry.ID) + if err != nil { + if isKnowledgeDownloadCanceled(err) { + return err + } + return errors.NewInternalServerError("部分文件无法读取,未生成压缩包,请检查文档后重试").WithDetails( + gin.H{"knowledge_id": entry.ID}, + ) + } + name := uniqueKnowledgeDownloadZipPath(entry.FolderPath, filename, usedNames) + // 原样存储可避免 PDF、Office 等已压缩文件再次压缩的 CPU 开销。 + zipEntry, err := writer.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store}) + if err != nil { + _ = file.Close() + return errors.NewInternalServerError("无法写入下载压缩包,请稍后重试") + } + n, copyErr := io.Copy(zipEntry, io.LimitReader( + &knowledgeDownloadReader{ctx: ctx, reader: file}, limit-total+1, + )) + closeErr := file.Close() + total += n + if total > limit { + return errors.NewBadRequestError("所选文件合计超过 512 MiB,请分批下载") + } + if isKnowledgeDownloadCanceled(copyErr) { + return copyErr + } + if copyErr != nil || closeErr != nil { + return errors.NewInternalServerError("部分文件读取失败,未生成压缩包,请稍后重试").WithDetails( + gin.H{"knowledge_id": entry.ID}, + ) + } + } + if err := writer.Close(); err != nil { + return errors.NewInternalServerError("无法完成下载压缩包,请稍后重试") + } + return nil +} + +// 每次读取前检查取消信号,用户离开页面后停止继续打包。 +type knowledgeDownloadReader struct { + ctx context.Context + reader io.Reader +} + +func (r *knowledgeDownloadReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +func uniqueKnowledgeDownloadName(filename string, used map[string]bool) string { + return uniqueKnowledgeDownloadZipPath("", filename, used) +} + +func uniqueKnowledgeDownloadZipPath(folderPath, filename string, used map[string]bool) string { + base := sanitizeDownloadFileName(filename) + ext := path.Ext(base) + stem := strings.TrimSuffix(base, ext) + dir := sanitizeDownloadFolderPath(folderPath) + name := base + if dir != "" { + name = dir + "/" + base + } + for index := 2; used[strings.ToLower(name)]; index++ { + suffixed := fmt.Sprintf("%s (%d)%s", stem, index, ext) + if dir != "" { + name = dir + "/" + suffixed + } else { + name = suffixed + } + } + used[strings.ToLower(name)] = true + return name +} + +func sanitizeDownloadFileName(filename string) string { + filename = path.Base(strings.ReplaceAll(filename, "\\", "/")) + filename = sanitizeDownloadRunes(filename) + filename = strings.Trim(filename, " .") + if filename == "" { + filename = "document" + } + ext := path.Ext(filename) + if len(ext) > 20 { + ext = "" + } + stem := strings.TrimSuffix(filename, ext) + stem = shortenDownloadName(stem, 180) + stem = strings.TrimRight(stem, " .") + if stem == "" { + stem = "document" + } + return escapeWindowsReservedName(stem) + ext +} + +func sanitizeDownloadFolderPath(folderPath string) string { + folderPath = strings.ReplaceAll(folderPath, "\\", "/") + var parts []string + var total int + for _, part := range strings.Split(folderPath, "/") { + part = sanitizeDownloadRunes(part) + part = strings.Trim(part, " .") + if part == "" || part == "." || part == ".." { + continue + } + part = shortenDownloadName(part, 80) + part = strings.TrimRight(part, " .") + if part == "" { + continue + } + part = escapeWindowsReservedName(part) + if total+len(part)+1 > 160 { + break + } + parts = append(parts, part) + total += len(part) + 1 + } + return strings.Join(parts, "/") +} + +func sanitizeDownloadRunes(value string) string { + return strings.Map(func(r rune) rune { + if unicode.IsControl(r) || strings.ContainsRune(`<>:"/\|?*`, r) { + return '_' + } + return r + }, value) +} + +func shortenDownloadName(value string, limit int) string { + var shortened strings.Builder + for _, r := range value { + if shortened.Len()+len(string(r)) > limit { + break + } + shortened.WriteRune(r) + } + return shortened.String() +} + +func escapeWindowsReservedName(stem string) string { + reserved := strings.ToUpper(strings.Split(stem, ".")[0]) + reservedRunes := []rune(reserved) + if reserved == "CON" || reserved == "PRN" || reserved == "AUX" || reserved == "NUL" || + (len(reservedRunes) == 4 && (strings.HasPrefix(reserved, "COM") || strings.HasPrefix(reserved, "LPT")) && + strings.ContainsRune("123456789¹²³", reservedRunes[3])) { + return "_" + stem + } + return stem +} diff --git a/internal/handler/knowledge_download_test.go b/internal/handler/knowledge_download_test.go new file mode 100644 index 00000000..9e834d44 --- /dev/null +++ b/internal/handler/knowledge_download_test.go @@ -0,0 +1,393 @@ +package handler + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "unicode/utf8" + + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type downloadKnowledgeStub struct { + interfaces.KnowledgeService + items []*types.Knowledge + names map[string]string + opened []string + failID string + expectedTenant uint64 +} + +func (s *downloadKnowledgeStub) GetKnowledgeBatch(context.Context, uint64, []string) ([]*types.Knowledge, error) { + return s.items, nil +} + +func (s *downloadKnowledgeStub) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) { + want := s.expectedTenant + if want == 0 { + want = 7 + } + if types.MustTenantIDFromContext(ctx) != want { + return nil, "", fmt.Errorf("错误的租户上下文") + } + s.opened = append(s.opened, id) + if s.failID == id { + return nil, "", fmt.Errorf("模拟存储读取失败") + } + return io.NopCloser(strings.NewReader("原文-" + id)), s.names[id], nil +} + +type downloadKBStub struct { + interfaces.KnowledgeBaseService + kb *types.KnowledgeBase +} + +func (s *downloadKBStub) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) { + return s.kb, nil +} + +type downloadShareStub struct { + interfaces.KBShareService + permission types.OrgMemberRole +} + +func (s *downloadShareStub) CheckTenantKBPermission( + context.Context, string, uint64, types.TenantRole, +) (types.OrgMemberRole, bool, error) { + return s.permission, true, nil +} + +func runBatchDownload( + t *testing.T, + svc *downloadKnowledgeStub, + ids []string, + kb *types.KnowledgeBase, + share interfaces.KBShareService, + scope *types.TenantAPIKeyScope, +) *httptest.ResponseRecorder { + t.Helper() + gin.SetMode(gin.TestMode) + router := gin.New() + router.Use(middleware.ErrorHandler(), func(c *gin.Context) { + ctx := types.WithCaller(c.Request.Context(), types.Caller{ + TenantID: 7, UserID: "user-1", Role: types.TenantRoleContributor, + }) + ctx = types.WithExecutionTenant(ctx, 7) + if scope != nil { + ctx = types.WithTenantAPIKeyScope(ctx, *scope) + } + c.Request = c.Request.WithContext(ctx) + c.Set(types.TenantIDContextKey.String(), uint64(7)) + c.Set(types.UserIDContextKey.String(), "user-1") + c.Next() + }) + h := &KnowledgeHandler{ + kgService: svc, + kbService: &downloadKBStub{kb: kb}, + kbShareService: share, + } + router.POST("/knowledge-bases/:id/knowledge/batch-download", h.BatchDownloadKnowledge) + body, err := json.Marshal(BatchDownloadKnowledgeRequest{IDs: ids}) + require.NoError(t, err) + req := httptest.NewRequest( + http.MethodPost, "/knowledge-bases/kb-1/knowledge/batch-download", bytes.NewReader(body), + ) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w +} + +func TestBatchDownloadKnowledgeProducesCompleteZIP(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + svc := &downloadKnowledgeStub{ + items: []*types.Knowledge{ + {ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-a"}, + {ID: "b", TenantID: 7, KnowledgeBaseID: "kb-1", Type: types.KnowledgeTypeManual}, + {ID: "c", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-c"}, + }, + names: map[string]string{"a": "设计说明.md", "b": "设计说明.md", "c": `..\设计说明 (2).md`}, + } + w := runBatchDownload(t, svc, []string{"a", "b", "a", "c"}, + &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + require.Equal(t, "application/zip", w.Header().Get("Content-Type")) + require.Equal(t, "private, no-store", w.Header().Get("Cache-Control")) + require.Contains(t, w.Header().Get("Content-Disposition"), "attachment") + reader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 3) + for index, name := range []string{"设计说明.md", "设计说明 (2).md", "设计说明 (2) (2).md"} { + entry := reader.File[index] + require.Equal(t, name, entry.Name) + file, err := entry.Open() + require.NoError(t, err) + content, err := io.ReadAll(file) + require.NoError(t, err) + require.NoError(t, file.Close()) + require.Equal(t, "原文-"+[]string{"a", "b", "c"}[index], string(content)) + } + require.Equal(t, []string{"a", "b", "c"}, svc.opened) + entries, err := os.ReadDir(os.TempDir()) + require.NoError(t, err) + require.Empty(t, entries, "请求完成后不应留下临时压缩包") +} + +func TestBatchDownloadKnowledgeRejectsInvalidSelectionsBeforeReading(t *testing.T) { + cases := []struct { + name string + ids []string + item *types.Knowledge + kb *types.KnowledgeBase + share interfaces.KBShareService + scope *types.TenantAPIKeyScope + status int + }{ + { + name: "空列表", status: 400, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "空白ID", ids: []string{" "}, status: 400, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "超出数量", ids: make([]string, maxBatchDownloadFiles+1), status: 400, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "文档缺失", ids: []string{"missing"}, status: 404, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "跨知识库", ids: []string{"a"}, status: 404, + item: &types.Knowledge{ID: "a", TenantID: 7, KnowledgeBaseID: "other", FilePath: "secret"}, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "跨租户", ids: []string{"a"}, status: 404, + item: &types.Knowledge{ID: "a", TenantID: 8, KnowledgeBaseID: "kb-1", FilePath: "secret"}, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "仅无原文件", ids: []string{"a"}, status: 400, + item: &types.Knowledge{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "url"}, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + }, + { + name: "共享只读", ids: []string{"a"}, status: 403, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 8}, + share: &downloadShareStub{permission: types.OrgRoleViewer}, + }, + { + name: "密钥无此库权限", ids: []string{"a"}, status: 403, + kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, + scope: &types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"other"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc := &downloadKnowledgeStub{} + if tc.item != nil { + svc.items = []*types.Knowledge{tc.item} + } + w := runBatchDownload(t, svc, tc.ids, tc.kb, tc.share, tc.scope) + require.Equal(t, tc.status, w.Code, w.Body.String()) + require.Empty(t, svc.opened) + require.NotContains(t, w.Header().Get("Content-Type"), "zip") + }) + } +} + +func TestBatchDownloadKnowledgeFailureDoesNotReturnPartialZIP(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + svc := &downloadKnowledgeStub{ + items: []*types.Knowledge{ + {ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "a"}, + {ID: "b", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "b"}, + }, + names: map[string]string{"a": "first.txt"}, failID: "b", + } + w := runBatchDownload(t, svc, []string{"a", "b"}, + &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil) + require.Equal(t, http.StatusInternalServerError, w.Code) + require.Contains(t, w.Header().Get("Content-Type"), "json") + require.Empty(t, w.Header().Get("Content-Disposition")) + entries, err := os.ReadDir(os.TempDir()) + require.NoError(t, err) + require.Empty(t, entries) +} + +type downloadTrackingReader struct { + io.Reader + closed bool +} + +func (r *downloadTrackingReader) Close() error { + r.closed = true + return nil +} + +func TestBatchDownloadKnowledgeSkipsEntriesWithoutOriginalFiles(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + svc := &downloadKnowledgeStub{ + items: []*types.Knowledge{ + {ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-a"}, + {ID: "url", TenantID: 7, KnowledgeBaseID: "kb-1", Type: "url"}, + }, + names: map[string]string{"a": "keep.txt"}, + } + w := runBatchDownload(t, svc, []string{"a", "url"}, + &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + reader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + require.Equal(t, "keep.txt", reader.File[0].Name) + require.Equal(t, []string{"a"}, svc.opened) +} + +func TestBatchDownloadKnowledgeAllowsSharedEditor(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + svc := &downloadKnowledgeStub{ + items: []*types.Knowledge{{ID: "a", TenantID: 8, KnowledgeBaseID: "kb-1", FilePath: "stored-a"}}, + names: map[string]string{"a": "shared.txt"}, + expectedTenant: 8, + } + w := runBatchDownload(t, svc, []string{"a"}, + &types.KnowledgeBase{ID: "kb-1", TenantID: 8}, + &downloadShareStub{permission: types.OrgRoleEditor}, nil) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + require.Equal(t, []string{"a"}, svc.opened) +} + +func TestBatchDownloadKnowledgeRejectsWhenBusy(t *testing.T) { + for i := 0; i < maxConcurrentBatchDownloads; i++ { + batchDownloadSlots <- struct{}{} + } + t.Cleanup(func() { + for i := 0; i < maxConcurrentBatchDownloads; i++ { + <-batchDownloadSlots + } + }) + svc := &downloadKnowledgeStub{ + items: []*types.Knowledge{{ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "a"}}, + names: map[string]string{"a": "a.txt"}, + } + w := runBatchDownload(t, svc, []string{"a"}, + &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil) + require.Equal(t, http.StatusTooManyRequests, w.Code, w.Body.String()) + require.Empty(t, svc.opened) +} + +func TestKnowledgeDownloadArchiveEnforcesActualByteLimitAndClosesFiles(t *testing.T) { + for _, limit := range []int64{4, 5} { + var output bytes.Buffer + file := &downloadTrackingReader{Reader: strings.NewReader("12345")} + err := writeKnowledgeDownloadArchive(context.Background(), &output, + []knowledgeDownloadEntry{{ID: "a"}}, + func(context.Context, string) (io.ReadCloser, string, error) { return file, "a.txt", nil }, limit) + if limit == 4 { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.True(t, file.closed) + } +} + +func TestKnowledgeDownloadArchiveStopsWhenCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var output bytes.Buffer + err := writeKnowledgeDownloadArchive(ctx, &output, []knowledgeDownloadEntry{{ID: "a"}}, + func(context.Context, string) (io.ReadCloser, string, error) { + t.Fatal("取消后不应继续读取文件") + return nil, "", nil + }, 100) + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, output.Len()) +} + +func TestKnowledgeDownloadArchiveCancelsAfterOpenAndClosesFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + file := &downloadTrackingReader{Reader: strings.NewReader("12345")} + err := writeKnowledgeDownloadArchive(ctx, io.Discard, []knowledgeDownloadEntry{{ID: "a"}}, + func(context.Context, string) (io.ReadCloser, string, error) { + cancel() + return file, "a.txt", nil + }, 100) + require.True(t, isKnowledgeDownloadCanceled(err), err) + require.True(t, file.closed) + requireAppHTTP(t, mapKnowledgeDownloadError(err), http.StatusBadRequest) +} + +func TestMapKnowledgeDownloadError(t *testing.T) { + requireAppHTTP(t, mapKnowledgeDownloadError(context.Canceled), http.StatusBadRequest) + requireAppHTTP(t, mapKnowledgeDownloadError(context.DeadlineExceeded), http.StatusBadRequest) +} + +func requireAppHTTP(t *testing.T, err error, status int) { + t.Helper() + appErr, ok := apperrors.IsAppError(err) + require.True(t, ok, err) + require.Equal(t, status, appErr.HTTPCode) +} + +func TestKnowledgeDownloadNamesAreSafeForWindowsAndUTF8(t *testing.T) { + used := map[string]bool{} + for _, name := range []string{ + "../报告.txt", `C:\secret\Report.TXT`, "report.txt", "CON.txt", "COM¹.txt", "a:b?.txt", "..", + strings.Repeat("中", 100) + ".pdf", + } { + got := uniqueKnowledgeDownloadName(name, used) + require.True(t, utf8.ValidString(got)) + require.NotContains(t, got, "/") + require.NotContains(t, got, "\\") + require.NotContains(t, got, ":") + require.NotEqual(t, "CON.txt", got) + require.NotEqual(t, "COM¹.txt", got) + require.Less(t, len(got), 255) + } + require.True(t, used["report (2).txt"]) +} + +func TestKnowledgeDownloadZipPathKeepsSafeFolders(t *testing.T) { + used := map[string]bool{} + require.Equal(t, "docs/spec/a.md", uniqueKnowledgeDownloadZipPath(`..\docs/spec`, "a.md", used)) + require.Equal(t, "docs/spec/a (2).md", uniqueKnowledgeDownloadZipPath("docs/spec", "../a.md", used)) + got := uniqueKnowledgeDownloadZipPath("/abs/../secret", "CON.txt", map[string]bool{}) + require.Equal(t, "abs/secret/_CON.txt", got) + require.NotContains(t, got, "..") + require.False(t, strings.HasPrefix(got, "/")) +} + +func TestBatchDownloadKnowledgePreservesFolderPaths(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + svc := &downloadKnowledgeStub{ + items: []*types.Knowledge{{ + ID: "a", TenantID: 7, KnowledgeBaseID: "kb-1", FilePath: "stored-a", FolderPath: "docs/spec", + }}, + names: map[string]string{"a": "design.md"}, + } + w := runBatchDownload(t, svc, []string{"a"}, + &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, nil, nil) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + reader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + require.Equal(t, "docs/spec/design.md", reader.File[0].Name) +} diff --git a/internal/router/router_api_key_capabilities_test.go b/internal/router/router_api_key_capabilities_test.go index 5c604c79..713a0b5d 100644 --- a/internal/router/router_api_key_capabilities_test.go +++ b/internal/router/router_api_key_capabilities_test.go @@ -307,6 +307,7 @@ func TestKnowledgeReadRoutesDeclareRetrieveCapability(t *testing.T) { {http.MethodGet, "/api/v1/knowledge-bases/:id"}, {http.MethodPost, "/api/v1/knowledge-bases/:id/hybrid-search"}, {http.MethodGet, "/api/v1/knowledge-bases/:id/knowledge"}, + {http.MethodPost, "/api/v1/knowledge-bases/:id/knowledge/batch-download"}, {http.MethodGet, "/api/v1/knowledge/:id"}, {http.MethodGet, "/api/v1/knowledge/:id/download"}, {http.MethodPost, "/api/v1/knowledge-bases/:id/faq/search"}, diff --git a/internal/router/router_knowledge_download_test.go b/internal/router/router_knowledge_download_test.go index 0d0438a2..678ff832 100644 --- a/internal/router/router_knowledge_download_test.go +++ b/internal/router/router_knowledge_download_test.go @@ -1,6 +1,7 @@ package router import ( + "bytes" "context" "net/http" "net/http/httptest" @@ -108,3 +109,45 @@ func TestKnowledgeDownloadRejectsReadOnlySharedKB(t *testing.T) { require.Equal(t, http.StatusForbidden, rec.Code, "body=%s", rec.Body.String()) } + +func TestBatchKnowledgeDownloadRejectsTenantViewer(t *testing.T) { + engine := newKnowledgeDownloadRouteTestEngine( + t, + types.TenantRoleViewer, + &types.Knowledge{ID: "knowledge-own", KnowledgeBaseID: "kb-own", TenantID: 1}, + &types.KnowledgeBase{ID: "kb-own", TenantID: 1}, + nil, + ) + + rec := httptest.NewRecorder() + req := httptest.NewRequest( + http.MethodPost, + "/api/v1/knowledge-bases/kb-own/knowledge/batch-download", + bytes.NewBufferString(`{"ids":["knowledge-own"]}`), + ) + req.Header.Set("Content-Type", "application/json") + engine.ServeHTTP(rec, req) + + require.Equal(t, http.StatusForbidden, rec.Code, "body=%s", rec.Body.String()) +} + +func TestBatchKnowledgeDownloadRejectsReadOnlySharedKB(t *testing.T) { + engine := newKnowledgeDownloadRouteTestEngine( + t, + types.TenantRoleContributor, + &types.Knowledge{ID: "knowledge-shared", KnowledgeBaseID: "kb-shared", TenantID: 2}, + &types.KnowledgeBase{ID: "kb-shared", TenantID: 2}, + &downloadKBShareStub{permission: types.OrgRoleViewer, source: 2}, + ) + + rec := httptest.NewRecorder() + req := httptest.NewRequest( + http.MethodPost, + "/api/v1/knowledge-bases/kb-shared/knowledge/batch-download", + bytes.NewBufferString(`{"ids":["knowledge-shared"]}`), + ) + req.Header.Set("Content-Type", "application/json") + engine.ServeHTTP(rec, req) + + require.Equal(t, http.StatusForbidden, rec.Code, "body=%s", rec.Body.String()) +} diff --git a/internal/router/routes_knowledge.go b/internal/router/routes_knowledge.go index e5a30516..c0dee2e1 100644 --- a/internal/router/routes_knowledge.go +++ b/internal/router/routes_knowledge.go @@ -74,6 +74,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl kb.POST("/url", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateKnowledgeFromURL) kb.POST("/manual", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateManualKnowledge) kbRead.GET("", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledge) + // 原始文件下载沿用单文件下载的 Contributor + Editor 权限边界。 + kbRead.POST("/batch-download", g.Contributor(), g.KBAccessWrite("id"), handler.BatchDownloadKnowledge) kbRead.GET("/folders", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledgeFolders) kb.PUT("/folders", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.RenameKnowledgeFolder) // Clearing all contents under a KB is a destructive op; gate