From db7046b85effa8104fed2f51a38cd65226c97b6d Mon Sep 17 00:00:00 2001 From: tayadwx <42323260+tayadwx@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:18:57 +0800 Subject: [PATCH] refactor(image): turn the hard-coded image flow into configurable pipelines (#3746) * feat(image): configurable image pipelines, rebuilt on current main Rebuilds the image-pipeline work on top of the current main, carrying only the pipeline and request-side changes: - The hard-coded image flow becomes registry-driven pipelines (default / smartocr), selectable per knowledge base, with a frontend panel and per-action thinking switches addressed as action-level keys (caption.thinking / ocr.thinking) declared once and carried into every pipeline. - Request-side support: VLM.PredictWithOptions carries a per-call thinking switch through every implementation and decorator, including the background concurrency wrapper. - OCR failure handling follows main's #4132 semantics: invalid output is rejected and recorded (ocr_status / ocr_error_code / .ocr subspan) without retrying the whole image; caption/observation failures get symmetric subspans instead of leaving the image span green. - Permanent VLM failures skip the asynq retry budget; a run that produced nothing fails honestly instead of passing as skipped. - UI and locale fixes travel with the panel. Concurrency, rate limiting and circuit breaking are NOT part of this branch: they stay with the existing system limiter. * fix(image): address pipeline configuration and execution regressions * fix(image): align pipeline overrides and validation semantics * style(image): keep the new pipeline code inside the lll and gofumpt gates --------- Co-authored-by: tayadwx Co-authored-by: wizardchen --- frontend/src/api/knowledge-base/index.ts | 71 + frontend/src/i18n/localeKeyAudit.ts | 6 + frontend/src/i18n/locales/en-US.ts | 58 + frontend/src/i18n/locales/ja-JP.ts | 58 + frontend/src/i18n/locales/ko-KR.ts | 58 + frontend/src/i18n/locales/ru-RU.ts | 58 + frontend/src/i18n/locales/zh-CN.ts | 55 + frontend/src/i18n/locales/zh-TW.ts | 54 +- frontend/src/types/knowledgeProcess.ts | 5 + .../src/utils/imageProcessingConfig.test.ts | 123 +- frontend/src/utils/imageProcessingConfig.ts | 81 ++ .../knowledge/ImagePipelineSettings.test.ts | 185 +++ .../views/knowledge/ImagePipelineSettings.vue | 334 +++++ .../knowledge/KnowledgeBaseEditorModal.vue | 91 +- .../components/UploadConfirmDialog.vue | 85 +- .../uploadImageVectorOverride.test.ts | 54 + go.sum | 1267 +++++++++++++++++ .../application/service/image_action_loop.go | 124 -- .../service/image_action_registry.go | 586 ++++++++ .../application/service/image_multimodal.go | 313 ++-- .../service/image_multimodal_attrs_test.go | 137 +- .../image_multimodal_skipretry_test.go | 211 +++ .../service/image_ocr_error_test.go | 162 +++ .../service/image_ocr_live_test.go | 2 +- .../service/image_ocr_validation_test.go | 24 +- .../service/image_pipeline_default.go | 117 ++ .../service/image_pipeline_golden_test.go | 496 +++++++ .../service/image_pipeline_overrides_test.go | 93 ++ .../service/image_pipeline_rules.go | 99 ++ .../service/image_pipeline_rules_test.go | 187 +++ .../service/image_pipeline_smartocr.go | 123 ++ .../service/image_pipeline_thinking_test.go | 177 +++ .../application/service/knowledge_process.go | 30 +- .../service/knowledge_process_config.go | 37 +- internal/application/service/knowledgebase.go | 12 + .../service/temporary_document_test.go | 14 + internal/handler/knowledgebase.go | 65 + internal/models/vlm/concurrency_wrapper.go | 11 +- internal/models/vlm/langfuse_wrapper.go | 10 +- internal/models/vlm/langfuse_wrapper_test.go | 97 ++ internal/models/vlm/llm_debug.go | 14 +- internal/models/vlm/ollama.go | 11 + internal/models/vlm/ollama_test.go | 66 + internal/models/vlm/remote_api.go | 27 +- internal/models/vlm/vlm.go | 22 +- internal/models/vlm/weknoracloud.go | 9 + internal/router/routes_knowledge.go | 12 + internal/types/image_actions.go | 288 +++- internal/types/knowledge_process.go | 10 + internal/types/knowledgebase.go | 27 + internal/types/task.go | 28 +- 51 files changed, 5825 insertions(+), 459 deletions(-) create mode 100644 frontend/src/views/knowledge/ImagePipelineSettings.test.ts create mode 100644 frontend/src/views/knowledge/ImagePipelineSettings.vue delete mode 100644 internal/application/service/image_action_loop.go create mode 100644 internal/application/service/image_action_registry.go create mode 100644 internal/application/service/image_multimodal_skipretry_test.go create mode 100644 internal/application/service/image_ocr_error_test.go create mode 100644 internal/application/service/image_pipeline_default.go create mode 100644 internal/application/service/image_pipeline_golden_test.go create mode 100644 internal/application/service/image_pipeline_overrides_test.go create mode 100644 internal/application/service/image_pipeline_rules.go create mode 100644 internal/application/service/image_pipeline_rules_test.go create mode 100644 internal/application/service/image_pipeline_smartocr.go create mode 100644 internal/application/service/image_pipeline_thinking_test.go create mode 100644 internal/models/vlm/langfuse_wrapper_test.go create mode 100644 internal/models/vlm/ollama_test.go diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 48e0bef1..74e9d52e 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -220,12 +220,83 @@ export interface ImageProcessingConfig { model_id?: string; image_actions?: ImageActionsConfig; image_attrs_enabled?: boolean; + /** Which image pipeline runs for this base; empty means "follow the switch". */ + image_pipeline?: string; + /** That pipeline's private tunables, keyed by field. See ImagePipelineField. */ + image_pipeline_params?: Record; // Index and recall images by their own vectors. Off unless the knowledge // base opts in, and only takes effect with an embedding model that takes // images. image_vector_enabled?: boolean; } +// Mirrors types.ImagePipelineSpec and types.ImageFieldDef. +// +// A field is private to the pipeline that declared it, which is why two +// pipelines may both own a field called "enable_ocr". The knowledge base stores +// them per pipeline rather than in one flat map, and the panel renders one +// control per field without naming any pipeline. +export interface ImagePipelineField { + key: string; + type: 'bool' | 'string' | 'enum'; + label: string; + description: string; + default: unknown; + options: string[] | null; + /** Whether this field turns an action on or off, as opposed to tuning one. */ + decides_action?: boolean; +} + +/** One condition the chosen pipeline places on its own tunables. The backend + * owns the rules — including the wording each one triggers — so the panel never + * decides on its own what a set of settings may or may not be. */ +export interface ImagePipelineRule { + at_least_one: string[]; + /** i18n key of this rule's own message, translated at render time. */ + message_key: string; + /** Which control to point at; defaults to the first of at_least_one. */ + field?: string; +} + +export interface ImagePipelineSpec { + id: string; + name: string; + /** How the pipeline works, shown under the pick; i18n overlays on top. */ + description: string; + fields: ImagePipelineField[]; + /** Conditions over those fields. Empty means the pipeline accepts any of it. */ + rules?: ImagePipelineRule[]; +} + +export interface ImagePipelineViolation { + message_key: string; + field?: string; +} + +/** POST /api/v1/image-pipelines/validate: every rule of the named pipeline that + * the given tunables break, translated by the caller rather than sent as prose. + * `valid: true` means the backend found none. */ +export async function validateImagePipelines( + pipelineId: string, + params: Record, +): Promise<{ valid: boolean; violations: ImagePipelineViolation[] }> { + const res = await post<{ + success: boolean; + data: { valid: boolean; violations: ImagePipelineViolation[] }; + }>(`/api/v1/image-pipelines/validate`, { pipeline_id: pipelineId, params }); + return res.data; +} + +// Fetch every registered image pipeline. The backend owns the list, so a +// pipeline added there shows up here as an option with no frontend change. The +// registry is global, not per-KB, hence the top-level route. +export async function fetchImagePipelines(): Promise { + const res = await get<{ success: boolean; data: ImagePipelineSpec[] }>( + `/api/v1/image-pipelines`, + ); + return res.data; +} + export type VectorStoreSource = 'env' | 'user' | 'shared' | 'unavailable'; export type VectorStoreStatus = 'available' | 'unavailable'; diff --git a/frontend/src/i18n/localeKeyAudit.ts b/frontend/src/i18n/localeKeyAudit.ts index 5e8a3a72..699b77d3 100644 --- a/frontend/src/i18n/localeKeyAudit.ts +++ b/frontend/src/i18n/localeKeyAudit.ts @@ -110,6 +110,12 @@ const EXTRA_PREFIXES = [ // so it is registered as a whole prefix: a new backend attribute must keep // being translated without touching the audit. 'imageAttr.', + // Image-pipeline panel text is keyed by the registry response too + // (imagePipeline.smartocr.label …), likewise runtime-built: a + // pipeline or field added in the backend must keep being translated without + // touching the audit. The panel's own wording is literal and lives in the + // locale files under knowledgeEditor.advanced.multimodal.imagePipeline*. + 'imagePipeline.', // Gallery attribute labels are keyed by the namespaced attribute id from // the gallery contract, likewise runtime-built; static labels for the few // builtin attributes ship in the locale files as diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 2727fd00..cad7c421 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -4015,6 +4015,14 @@ export default { imageVectorModelUnsupported: 'The current embedding model does not declare image input, so no image vectors will be indexed or recalled', imageAttrsSchemaLabel: 'Observable image attributes', imageAttrsSchemaDescription: 'The model observes the attributes below (defined by the backend registry) to drive the OCR policy', + imagePipelineSectionLabel: 'Image parsing scheme', + imagePipelineSectionDescription: + 'With multimodal parsing on, pick how images are parsed.', + imagePipelineLabel: 'Scheme', + imagePipelinePlaceholder: 'Select a scheme', + imagePipelineLoading: 'Loading pipelines…', + imagePipelineLoadError: 'Could not load pipelines', + imagePipelineValidateError: 'The current settings could not be checked; the backend re-checks them on save.', imageAttrsOcrConditions: 'Trigger OCR based on the observed attribute conditions', imageAttrsOcrConditionsDesc: 'When the observed attributes match the conditions below, OCR runs on the image', imageAttrsOcrOnUnobserved: 'Run OCR when image-attribute observation fails', @@ -8113,5 +8121,55 @@ export default { 'false': { label: 'No', description: 'a photo, drawing, icon or decoration' } } } + }, + // Overlay wording for the pipeline panel, indexed by pipeline id and field + // key (the backend registry supplies both; this only translates). Anything + // not covered here falls back to the backend's own wording, so a new + // pipeline never shows up as a bare key. + imagePipeline: { + errors: { + noActionEnabled: 'Turn on at least one parsing action, otherwise nothing will be processed.', + }, + default: { + name: 'Manual', + description: 'Turn the parsing actions on or off to fit the task.', + enable_caption: { + label: 'Image caption', + description: 'Ask the model for a one-line description of every image and store it as the caption.' + }, + enable_ocr: { + label: 'OCR', + description: 'Extract the text that appears in the image.' + }, + caption: { + thinking: { + label: 'Caption with thinking', + description: 'Let the model think before it writes the caption. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the caption, failing the task. Only enable it when you have confirmed it is necessary.', + }, + }, + ocr: { + thinking: { + label: 'OCR with thinking', + description: 'Let the model think before it transcribes the text. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the transcribed text, failing the task. Only enable it when you have confirmed it is necessary.', + }, + } + }, + smartocr: { + name: 'Smart', + caption: { + thinking: { + label: 'Describe with thinking', + description: 'Let the model think before it describes the image and reports its attributes. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the description, failing the task. Only enable it when you have confirmed it is necessary.', + }, + }, + ocr: { + thinking: { + label: 'Text recognition with thinking', + description: 'Let the model think before it transcribes the text. In general you do not need thinking; enabling it makes parsing slower and may make the model overthink, so overly long reasoning can run past the limit and truncate the transcribed text, failing the task. Only enable it when you have confirmed it is necessary.', + }, + }, + description: + 'Observe the image attributes and describe the image first, then decide from the attributes whether to run OCR, saving model calls and speeding up parsing.' + } } } diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts index 48fe073a..9d474676 100644 --- a/frontend/src/i18n/locales/ja-JP.ts +++ b/frontend/src/i18n/locales/ja-JP.ts @@ -4015,6 +4015,14 @@ export default { imageVectorModelUnsupported: '現在の埋め込みモデルは画像入力を宣言していないため、画像ベクトルは生成も検索もされません', imageAttrsSchemaLabel: '観察可能な画像属性', imageAttrsSchemaDescription: 'モデルは以下の属性(バックエンドのレジストリで定義)を観察して OCR ポリシーを決めます', + imagePipelineSectionLabel: '画像解析方式', + imagePipelineSectionDescription: + 'マルチモーダル解析が有効な場合、画像の解析方式を選択できます。', + imagePipelineLabel: '解析方式', + imagePipelinePlaceholder: '解析方式を選択', + imagePipelineLoading: 'パイプラインを読み込み中…', + imagePipelineLoadError: 'パイプラインの読み込みに失敗しました', + imagePipelineValidateError: '現在の設定を検証できませんでした。保存時にバックエンドが再度检查します。', imageAttrsOcrConditions: '観察した属性条件に基づいて OCR を実行', imageAttrsOcrConditionsDesc: '観察した属性が以下の条件を満たす場合、その画像に OCR を実行します', imageAttrsOcrOnUnobserved: '画像属性の観察に失敗した場合も OCR を実行', @@ -8107,5 +8115,55 @@ export default { 'false': { label: 'いいえ', description: 'いいえ —— 写真・イラスト・アイコン・装飾' } } } + }, + // パイプラインパネルの上書き文言:パイプライン id とフィールド key で索引 + // (どちらもバックエンドのレジストリが提供し、ここでは翻訳のみ)。ここに + // ない部分はバックエンド固有の文言にフォールバックするため、新しい + // パイプラインがキー名のまま表示されることはありません。 + imagePipeline: { + errors: { + noActionEnabled: '解析アクションを1つ以上オンにしてください。オフのままでは何も処理されません。', + }, + default: { + name: 'マニュアル', + description: 'タスクに合わせて解析アクションのスイッチを手動でオン/オフします。', + enable_caption: { + label: '画像説明', + description: '各画像について一言の説明を生成し、キャプションとして保存します。' + }, + enable_ocr: { + label: '文字認識', + description: '画像内に出現するテキストを抽出します。' + }, + caption: { + thinking: { + label: '説明時に思考を有効', + description: '説明を書く前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。', + }, + }, + ocr: { + thinking: { + label: '文字認識時に思考を有効', + description: 'テキストを抽出する前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。', + }, + } + }, + smartocr: { + name: 'スマート', + caption: { + thinking: { + label: '説明時に思考を有効', + description: '画像を説明し属性を報告する前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。', + }, + }, + ocr: { + thinking: { + label: '文字認識時に思考を有効', + description: 'テキストを抽出する前にモデルに考えさせます。基本的に思考は必要ありません。有効にすると解析が遅くなるだけでなく、思考が長くなりすぎて本文が切り詰められ、タスクが失敗する可能性があります。必要だと判断した場合のみ有効にしてください。', + }, + }, + description: + 'まず画像の特徴を観察して説明を生成し、その特徴に基づいて OCR を実行するかどうかを判断します。モデル呼び出しを節約し、解析速度を向上させます。' + } } } diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 8ff116a0..e0eb66cc 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -4362,6 +4362,14 @@ export default { imageVectorModelUnsupported: '현재 임베딩 모델은 이미지 입력을 선언하지 않아 이미지 벡터가 생성되거나 검색되지 않습니다', imageAttrsSchemaLabel: '관찰 가능한 이미지 속성', imageAttrsSchemaDescription: '모델은 아래 속성(백엔드 레지스트리 정의)을 관찰해 OCR 정책을 결정합니다', + imagePipelineSectionLabel: '이미지 분석 방식', + imagePipelineSectionDescription: + '멀티모달 분석이 켜져 있으면 이미지의 분석 방식을 필요에 따라 선택할 수 있습니다.', + imagePipelineLabel: '분석 방식', + imagePipelinePlaceholder: '분석 방식 선택', + imagePipelineLoading: '파이프라인을 불러오는 중…', + imagePipelineLoadError: '파이프라인을 불러오지 못했습니다', + imagePipelineValidateError: '현재 설정을 검증할 수 없습니다. 저장 시 백엔드가 다시 검사합니다.', imageAttrsOcrConditions: '관찰된 속성 조건에 따라 OCR 실행', imageAttrsOcrConditionsDesc: '관찰된 속성이 아래 조건을 충족하면 해당 이미지에 OCR을 실행합니다', imageAttrsOcrOnUnobserved: '이미지 속성 관찰에 실패해도 OCR 실행', @@ -8107,5 +8115,55 @@ export default { 'false': { label: '아니오', description: '아니오 —— 사진, 삽화, 아이콘, 장식' } } } + }, + // 파이프라인 패널의 오버레이 문구: 파이프라인 id와 필드 key로 색인합니다 + // (둘 다 백엔드 레지스트리가 제공하며 여기서는 번역만 합니다). 여기서 + // 다루지 않은 부분은 백엔드 자체 문구로 폴백되므로, 새 파이프라인이 키 + // 이름 그대로 표시되는 일은 없습니다. + imagePipeline: { + errors: { + noActionEnabled: '분석 동작을 하나 이상 켜세요. 모두 꺼져 있으면 아무것도 처리되지 않습니다.', + }, + default: { + name: '수동', + description: '작업에 맞게 분석 동작 스위치를 직접 켜거나 끕니다.', + enable_caption: { + label: '이미지 설명', + description: '각 이미지에 대한 한 줄 설명을 생성해 캡션으로 저장합니다.' + }, + enable_ocr: { + label: '문자 인식', + description: '이미지 속에 나타나는 텍스트를 추출합니다.' + }, + caption: { + thinking: { + label: '설명 시 사고 사용', + description: '설명을 쓰기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.', + }, + }, + ocr: { + thinking: { + label: 'OCR 시 사고 사용', + description: '텍스트를 추출하기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.', + }, + } + }, + smartocr: { + name: '스마트', + caption: { + thinking: { + label: '설명 시 사고 사용', + description: '이미지를 설명하고 속성을 보고하기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.', + }, + }, + ocr: { + thinking: { + label: '문자 인식 시 사고 사용', + description: '텍스트를 추출하기 전에 모델이 먼저 추론하도록 합니다. 일반적으로 사고는 필요하지 않습니다. 켜면 분석이 느려질 뿐 아니라 사고가 너무 길어져 본문이 잘려서 작업이 실패할 수 있습니다. 꼭 필요하다고 판단될 때만 켜세요.', + }, + }, + description: + '먼저 이미지 특징을 관찰하여 설명을 생성하고, 그 특징을 바탕으로 OCR 실행 여부를 결정합니다. 모델 호출을 절약하고 분석 속도를 높입니다.' + } } } diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 9608ff93..8d4216c5 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -4362,6 +4362,14 @@ export default { imageVectorModelUnsupported: 'Текущая модель эмбеддингов не заявляет поддержку изображений, поэтому векторы изображений не будут ни созданы, ни найдены', imageAttrsSchemaLabel: 'Наблюдаемые атрибуты изображений', imageAttrsSchemaDescription: 'Модель наблюдает перечисленные ниже атрибуты (определены реестром бэкенда), чтобы управлять политикой OCR', + imagePipelineSectionLabel: 'Схема разбора изображений', + imagePipelineSectionDescription: + 'Когда включён мультимодальный разбор, можно выбрать способ разбора изображений.', + imagePipelineLabel: 'Способ разбора', + imagePipelinePlaceholder: 'Выберите способ разбора', + imagePipelineLoading: 'Загрузка пайплайнов…', + imagePipelineLoadError: 'Не удалось загрузить пайплайны', + imagePipelineValidateError: 'Текущие настройки не удалось проверить; бэкенд проверит их при сохранении.', imageAttrsOcrConditions: 'Запуск OCR по наблюдаемым условиям атрибутов', imageAttrsOcrConditionsDesc: 'Когда наблюдаемые атрибуты соответствуют условиям ниже, для изображения выполняется OCR', imageAttrsOcrOnUnobserved: 'Запускать OCR при сбое наблюдения атрибутов изображения', @@ -8107,5 +8115,55 @@ export default { 'false': { label: 'Нет', description: 'нет —— фото, рисунок, значок или декор' } } } + }, + // Текст панели пайплайнов: индексируется по id пайплайна и ключу поля + // (оба даёт реестр бэкенда, здесь только перевод). Всё, что здесь не + // указано, откатывается к формулировке бэкенда, поэтому новый пайплайн + // не покажется голым именем ключа. + imagePipeline: { + errors: { + noActionEnabled: 'Включите хотя бы одно действие разбора, иначе ничего не будет обработано.', + }, + default: { + name: 'Ручной', + description: 'Пользователь сам включает или выключает действия разбора под задачу.', + enable_caption: { + label: 'Описание изображения', + description: 'Генерирует однострочное описание каждого изображения и сохраняет его как подпись.' + }, + enable_ocr: { + label: 'Распознавание текста', + description: 'Извлекает текст, присутствующий на изображении.' + }, + caption: { + thinking: { + label: 'С размышлением при описании', + description: 'Дать модели поразмышлять перед тем, как писать описание. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.', + }, + }, + ocr: { + thinking: { + label: 'С размышлением при распознавании', + description: 'Дать модели поразмышлять перед расшифровкой текста. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.', + }, + } + }, + smartocr: { + name: 'Умный', + caption: { + thinking: { + label: 'С размышлением при описании', + description: 'Дать модели поразмышлять перед описанием изображения и отчётом о его признаках. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.', + }, + }, + ocr: { + thinking: { + label: 'С размышлением при распознавании текста', + description: 'Дать модели поразмышлять перед расшифровкой текста. Как правило, в этом нет необходимости: включение размышлений замедляет обработку и слишком длинные размышления могут вытеснить основной текст, из-за чего задача завершится неудачей. Включайте только если вы уверены, что это необходимо.', + }, + }, + description: + 'Сначала наблюдаются признаки изображения и создаётся его описание, затем по этим признакам решается, вызывать ли OCR. Экономит вызовы модели и ускоряет разбор.' + } } } diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 384d5220..01c3a84c 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -4364,6 +4364,13 @@ export default { imageVectorModelUnsupported: '当前向量模型未声明图片输入,开启后不会生成或召回图片向量', imageAttrsSchemaLabel: '可观察的图片属性', imageAttrsSchemaDescription: '模型会观察以下属性(由后端注册表定义)以驱动 OCR 策略', + imagePipelineSectionLabel: '图片解析方案', + imagePipelineSectionDescription: '多模态解析开启后,可以根据需要选择图片的解析方案', + imagePipelineLabel: '选择解析方案', + imagePipelinePlaceholder: '请选择解析方案', + imagePipelineLoading: '正在读取流水线…', + imagePipelineLoadError: '读取流水线失败', + imagePipelineValidateError: '无法校验当前设置,保存时后端会再次检查。', imageAttrsOcrConditions: '根据观察到的属性条件触发 OCR', imageAttrsOcrConditionsDesc: '当观察到的属性满足以下条件时,对图片进行 OCR', imageAttrsOcrOnUnobserved: '图片属性观察失败时仍执行 OCR', @@ -8109,5 +8116,53 @@ export default { 'false': { label: '否', description: '否 —— 照片、插画、图标或装饰图' } } } + }, + // 流水线选择面板的覆盖文案:按流水线 id 与字段 key 索引(后端注册表给出 + // id 与 key,这里只做翻译)。未覆盖的流水线或字段回落到后端自带的英文 + // 说明,所以新增流水线不会显示成键名。 + imagePipeline: { + errors: { + noActionEnabled: '至少开启一个解析动作,否则多模态不会处理任何内容。', + }, + default: { + name: '传统', + description: '用户根据任务需要手动选择解析动作开关。', + enable_caption: { + label: '图片描述', + description: '让模型为每张图片生成一句内容描述,作为图片的说明文字。' + }, + enable_ocr: { + label: '文字识别', + description: '提取图片中出现的文字。' + }, + caption: { + thinking: { + label: '图片描述时开启思考', + description: '让模型在写描述前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。', + }, + }, + ocr: { + thinking: { + label: '文字识别时开启思考', + description: '让模型在提取文字前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。', + }, + } + }, + smartocr: { + name: '智能模式', + caption: { + thinking: { + label: '描述时开启思考', + description: '让模型在描述图片并观察属性前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。', + }, + }, + ocr: { + thinking: { + label: '文字识别时开启思考', + description: '让模型在提取文字前先思考。一般情况下不需要开启思考,开启之后不仅更慢,还可能造成大模型思考内容过长截断正文输出造成任务失败。请仅在确认有必要的情况下开启。', + }, + }, + description: '先观察图片特征并进行图片描述,再根据图片特征决定是否调用OCR对图片进行识别,以节省模型调用次数并提升解析速度。' + } } } diff --git a/frontend/src/i18n/locales/zh-TW.ts b/frontend/src/i18n/locales/zh-TW.ts index d8c5875b..68a3931d 100644 --- a/frontend/src/i18n/locales/zh-TW.ts +++ b/frontend/src/i18n/locales/zh-TW.ts @@ -4368,6 +4368,13 @@ export default { imageAttrsOcrConditionsDesc: '當觀察到的屬性滿足以下條件時,對圖片進行 OCR', imageAttrsOcrOnUnobserved: '圖片屬性觀察失敗時仍執行 OCR', imageAttrsOcrOnUnobservedDesc: '當模型未能正確觀察到圖片屬性時,預設仍執行 OCR 兜底,以免漏掉正文文字;關閉則跳過。(採用 4B 等小參數視覺模型,或自定義的圖片解析提示語與系統提示語衝突時,可能造成觀察失敗;8B 及以上模型的失敗概率很低,不建議關閉)', + imagePipelineSectionLabel: '圖片解析方案', + imagePipelineSectionDescription: '多模態解析開啟後,可以根據需要選擇圖片的解析方案', + imagePipelineLabel: '選擇解析方案', + imagePipelinePlaceholder: '請選擇解析方案', + imagePipelineLoading: '正在讀取流水線…', + imagePipelineLoadError: '讀取流水線失敗', + imagePipelineValidateError: '無法校驗當前設定,儲存時後端會再次檢查。', imagePipelineKbNote: '預設跟隨知識庫設定,可針對本次任務調整' }, tableMetadataInstructions: { @@ -8109,5 +8116,50 @@ export default { 'false': { label: '否', description: '否 ── 照片、插畫、圖示或裝飾圖' } } } - } + }, + imagePipeline: { + errors: { + noActionEnabled: '至少開啟一個解析動作,否則多模態不會處理任何內容。', + }, + default: { + name: '傳統', + description: '使用者根據任務需要手動選擇解析動作開關。', + enable_caption: { + label: '圖片描述', + description: '讓模型為每張圖片生成一句內容描述,作為圖片的說明文字。', + }, + enable_ocr: { + label: '文字辨識', + description: '提取圖片中出現的文字。', + }, + caption: { + thinking: { + label: '圖片描述時開啟思考', + description: '讓模型在寫描述前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。', + }, + }, + ocr: { + thinking: { + label: '文字辨識時開啟思考', + description: '讓模型在提取文字前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。', + }, + }, + }, + smartocr: { + name: '智能模式', + caption: { + thinking: { + label: '描述時開啟思考', + description: '讓模型在描述圖片並觀察屬性前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。', + }, + }, + ocr: { + thinking: { + label: '文字辨識時開啟思考', + description: '讓模型在提取文字前先思考。一般情況下不需要開啟思考,開啟之後不僅更慢,還可能造成大模型思考內容過長截斷正文輸出造成任務失敗。請僅在確認有必要的情況下開啟。', + }, + }, + description: '先觀察圖片特徵並進行圖片描述,再根據圖片特徵決定是否呼叫 OCR 對圖片進行辨識,以節省模型呼叫次數並提升解析速度。', + }, + }, } diff --git a/frontend/src/types/knowledgeProcess.ts b/frontend/src/types/knowledgeProcess.ts index 45f1af0b..53bec92d 100644 --- a/frontend/src/types/knowledgeProcess.ts +++ b/frontend/src/types/knowledgeProcess.ts @@ -74,6 +74,11 @@ export interface KnowledgeProcessOverrides { // Per-upload override of the knowledge base's attribute-observed image // pipeline switch. image_attrs_enabled?: boolean + // Per-upload override of the image pipeline and its private tunables. The + // pair travels as one unit; when image_pipeline is set the backend also + // derives image_attrs_enabled from it. + image_pipeline?: string + image_pipeline_params?: Record // Controls indexing for this document, without changing KB retrieval. image_vector_enabled?: boolean // Per-task attribute -> work table; merged per action key on top of the diff --git a/frontend/src/utils/imageProcessingConfig.test.ts b/frontend/src/utils/imageProcessingConfig.test.ts index 340aadb8..5fb336e9 100644 --- a/frontend/src/utils/imageProcessingConfig.test.ts +++ b/frontend/src/utils/imageProcessingConfig.test.ts @@ -1,48 +1,93 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { buildImageProcessingConfig } from './imageProcessingConfig.ts' +import { + buildImageProcessingConfig, + buildPipelineFields, + resolveImagePipelineFromKb, + normalizeImagePipelineId, + IMAGE_PIPELINE_SMARTOCR, + LEGACY_PIPELINE_CAPTION_OCR, + LEGACY_PIPELINE_OB_CAP_OCR, + type ImageProcessingEdits, +} from './imageProcessingConfig.ts' const DEFAULT_ON = [ { prop: 'contain.text', is: 'block' }, { prop: 'contain.data_visual', is: 'true' }, ] +/** The edits every test shares; `edits` overrides them. */ +function editsWith(overrides: Partial = {}): ImageProcessingEdits { + return { + imageAttrsEnabled: true, + imageVectorEnabled: false, + onUnobserved: false, + defaultOn: DEFAULT_ON, + pipelineId: IMAGE_PIPELINE_SMARTOCR, + pipelineParams: {}, + ...overrides, + } +} + test('saving keeps OCR conditions customised through the API', () => { // Regression: the editor always wrote the registry default into `on`, so an // unrelated edit replaced a KB's custom OCR conditions with the default table. const custom = [{ prop: 'contain.text', is: 'sparse' }] const built = buildImageProcessingConfig( { model_id: 'vlm-1', image_attrs_enabled: true, image_actions: { ocr: { on: custom, on_unobserved: true } } }, - { imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: false, defaultOn: DEFAULT_ON }, + editsWith(), ) assert.deepEqual(built, { model_id: 'vlm-1', image_attrs_enabled: true, image_actions: { ocr: { on: custom, on_unobserved: false } }, + image_pipeline: IMAGE_PIPELINE_SMARTOCR, }) }) test('a KB without custom conditions gets the registry default alongside on_unobserved', () => { const built = buildImageProcessingConfig( { model_id: 'vlm-1' }, - { imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: false, defaultOn: DEFAULT_ON }, + editsWith(), ) assert.deepEqual(built, { model_id: 'vlm-1', image_attrs_enabled: true, image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: false } }, + image_pipeline: IMAGE_PIPELINE_SMARTOCR, + }) +}) + +test('the chosen pipeline and its private parameters are written back', () => { + const built = buildImageProcessingConfig( + { model_id: 'vlm-1' }, + editsWith({ pipelineId: LEGACY_PIPELINE_CAPTION_OCR, pipelineParams: { enable_caption: false, enable_ocr: true } }), + ) + + // Both keys of this pipeline travel, and they are capitalised exactly as the + // backend declares them: a misspelled key is silently ignored at run time. + assert.deepEqual(built, { + model_id: 'vlm-1', + image_attrs_enabled: true, + image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: false } }, + image_pipeline: LEGACY_PIPELINE_CAPTION_OCR, + image_pipeline_params: { enable_caption: false, enable_ocr: true }, }) }) test('an unchanged configuration is not sent', () => { const custom = [{ prop: 'contain.text', is: 'sparse' }] - const snapshot = { image_attrs_enabled: true, image_actions: { ocr: { on: custom, on_unobserved: true } } } + const snapshot = { + image_attrs_enabled: true, + image_actions: { ocr: { on: custom, on_unobserved: true } }, + image_pipeline: IMAGE_PIPELINE_SMARTOCR, + } assert.equal( - buildImageProcessingConfig(snapshot, { imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: true, defaultOn: DEFAULT_ON }), + buildImageProcessingConfig(snapshot, editsWith({ onUnobserved: true })), null, ) }) @@ -50,13 +95,13 @@ test('an unchanged configuration is not sent', () => { test('the image-vector switch is saved when turned on and when turned back off', () => { const on = buildImageProcessingConfig( { model_id: 'vlm-1' }, - { imageAttrsEnabled: false, imageVectorEnabled: true, onUnobserved: true, defaultOn: DEFAULT_ON }, + editsWith({ imageAttrsEnabled: false, imageVectorEnabled: true }), ) assert.equal(on?.image_vector_enabled, true) const off = buildImageProcessingConfig( { model_id: 'vlm-1', image_vector_enabled: true }, - { imageAttrsEnabled: false, imageVectorEnabled: false, onUnobserved: true, defaultOn: DEFAULT_ON }, + editsWith({ imageAttrsEnabled: false, imageVectorEnabled: false }), ) assert.equal(off?.image_vector_enabled, false) }) @@ -64,11 +109,67 @@ test('the image-vector switch is saved when turned on and when turned back off', test('a knowledge base that never touched the image-vector switch does not get one written', () => { // The backend reads a missing key as off, which is what every knowledge // base from before the switch has; an unrelated save must leave it so. - const snapshot = { image_attrs_enabled: true, image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: true } } } + const snapshot = { + image_attrs_enabled: true, + image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: true } }, + image_pipeline: IMAGE_PIPELINE_SMARTOCR, + } assert.equal( - buildImageProcessingConfig(snapshot, { - imageAttrsEnabled: true, imageVectorEnabled: false, onUnobserved: true, defaultOn: DEFAULT_ON, - }), + buildImageProcessingConfig(snapshot, editsWith({ onUnobserved: true })), null, ) }) + +test('clearing the pick removes the stored one rather than repeating it', () => { + // An empty pick means "resolve from image_attrs_enabled", which is what the + // field already said by being absent, so saving has to actually drop it — + // along with the parameters, which belong to the pick that is gone. + const snapshot = { + image_pipeline: IMAGE_PIPELINE_SMARTOCR, + image_pipeline_params: { capture_caption: true }, + } + const built = buildImageProcessingConfig(snapshot, editsWith({ + pipelineId: '', + pipelineParams: { enable_caption: false, enable_ocr: false }, + })) + + assert.deepEqual(built, { + image_attrs_enabled: true, + image_actions: { ocr: { on: DEFAULT_ON, on_unobserved: false } }, + }) +}) + +test('both dialogs drop an empty pick and empty parameters the same way', () => { + // Regression: the upload dialog used to post `image_pipeline_params: {}`, + // which reads as "the caller turned everything off" once a pipeline starts + // reading its own keys with a default pinned at the read site. + // A real pick with nothing to tune keeps the pick and drops the params; + // an absent pick falls back to the backend's own resolution. + assert.deepEqual(buildPipelineFields(IMAGE_PIPELINE_SMARTOCR, {}), { + image_pipeline: IMAGE_PIPELINE_SMARTOCR, + }) + assert.deepEqual(buildPipelineFields('', { enable_ocr: true }), {}) +}) + +test('stored renames resolve to the ids the panel offers', () => { + // Regression: `caption_ocr` predates the picker and `ob_cap_ocr` predates + // the rename to match the label; a reparse of a document saved under either + // spelling must land on a selectable option, not on an empty one. + assert.equal(normalizeImagePipelineId(LEGACY_PIPELINE_CAPTION_OCR), 'default') + assert.equal(normalizeImagePipelineId(LEGACY_PIPELINE_OB_CAP_OCR), IMAGE_PIPELINE_SMARTOCR) + assert.equal( + resolveImagePipelineFromKb({ image_processing_config: { image_pipeline: LEGACY_PIPELINE_OB_CAP_OCR } }), + IMAGE_PIPELINE_SMARTOCR, + ) + // Only the observation switch, no pick at all: the legacy rule still applies. + assert.equal( + resolveImagePipelineFromKb({ image_processing_config: { image_attrs_enabled: true } }), + IMAGE_PIPELINE_SMARTOCR, + ) + assert.equal( + resolveImagePipelineFromKb({ image_processing_config: { image_attrs_enabled: false } }), + 'default', + ) + // Nothing stored at all — the oldest knowledge bases. + assert.equal(resolveImagePipelineFromKb({}), 'default') +}) diff --git a/frontend/src/utils/imageProcessingConfig.ts b/frontend/src/utils/imageProcessingConfig.ts index 0002992e..6ffa4448 100644 --- a/frontend/src/utils/imageProcessingConfig.ts +++ b/frontend/src/utils/imageProcessingConfig.ts @@ -4,6 +4,13 @@ export interface ImageAttrConditionLike { is: string } +/** Pipeline ids. Both dialogs compare against these instead of literals. */ +export const IMAGE_PIPELINE_DEFAULT = 'default' +export const IMAGE_PIPELINE_SMARTOCR = 'smartocr' +/** Spellings a stored config may still carry; normalized, never honoured. */ +export const LEGACY_PIPELINE_CAPTION_OCR = 'caption_ocr' +export const LEGACY_PIPELINE_OB_CAP_OCR = 'ob_cap_ocr' + /** The fields of the KB editor that feed image_processing_config. */ export interface ImageProcessingEdits { imageAttrsEnabled: boolean @@ -12,6 +19,53 @@ export interface ImageProcessingEdits { onUnobserved: boolean /** The registry's current default OCR conditions (GET /image-attrs/schema). */ defaultOn: ImageAttrConditionLike[] + /** The pipeline picked in the panel; empty means multimodal is off and the fields go away. */ + pipelineId: string + /** That pipeline's private tunables, keyed by field. */ + pipelineParams: Record +} + +/** The pipeline fields an overrides payload carries, with the empties dropped. */ +export interface PipelineFields { + image_pipeline?: string + image_pipeline_params?: Record +} + +/** + * Maps a stored pipeline id onto the id the panel offers. Both renames are + * folded in here so the KB editor, the upload dialog and the reparse dialog + * agree on what a saved value means; unknown ids pass through and the caller + * falls back. Mirrors backend types.NormalizeImagePipelineID. + */ +export function normalizeImagePipelineId(id: string | null | undefined): string { + if (!id) return '' + if (id === LEGACY_PIPELINE_CAPTION_OCR) return IMAGE_PIPELINE_DEFAULT + if (id === LEGACY_PIPELINE_OB_CAP_OCR) return IMAGE_PIPELINE_SMARTOCR + return id +} + +/** + * Builds the pipeline half of a payload. An empty pick is not a value of its + * own: the backend resolves from image_attrs_enabled when the field is absent, + * so the pair is dropped rather than written as an empty pair. The same goes + * for parameters — a pick with nothing to tune must not send an empty object, + * which would read as "the caller turned everything off". + * + * This is the only place that decides emptiness, so the knowledge base editor + * (whole-config replacement) and the upload/reparse dialogs (per-task + * overrides) cannot drift apart on what "no setting" means. + */ +export function buildPipelineFields( + pipelineId: string, + pipelineParams: Record | null | undefined, +): PipelineFields { + if (!pipelineId) return {} + const fields: PipelineFields = {} + if (pipelineId) fields.image_pipeline = pipelineId + if (pipelineParams && Object.keys(pipelineParams).length > 0) { + fields.image_pipeline_params = pipelineParams + } + return fields } /** @@ -25,6 +79,9 @@ export interface ImageProcessingEdits { * through the API keeps it; one without a custom list gets the registry's * current default, so it still follows the default as it evolves. * + * The pipeline and its parameters are written too. They are the panel's own + * settings; the observation switch is kept in step with the pick so a base that + * was edited through the UI still reads the same way through the older switch. * image_vector_enabled is written only when it is on or was already stored: * a knowledge base that never touched the switch reads as off on the * backend, so writing `false` into it would be a change for nothing. @@ -46,8 +103,32 @@ export function buildImageProcessingConfig( }, }, } + // The pick carries the panel's own answer, so it replaces whatever the + // snapshot had — including by removing it. + const pipeline = buildPipelineFields(edits.pipelineId, edits.pipelineParams) + if (pipeline.image_pipeline) built.image_pipeline = pipeline.image_pipeline + else delete built.image_pipeline + if (pipeline.image_pipeline_params) built.image_pipeline_params = pipeline.image_pipeline_params + else delete built.image_pipeline_params if (edits.imageVectorEnabled || 'image_vector_enabled' in snap) { built.image_vector_enabled = edits.imageVectorEnabled } return JSON.stringify(built) === JSON.stringify(snap) ? null : built } + +/** + * Resolves the pipeline id a knowledge base's stored config points at, by the + * same rule the backend's ResolveImagePipelineID follows: an explicitly stored + * id wins (a pre-rename "caption_ocr" normalizes to "default", "ob_cap_ocr" to + * "smartocr"), otherwise the legacy observation switch picks smartocr when on + * and default otherwise. Bases saved before the selector existed have no + * config at all and land on default, so the picker always shows a real option. + */ +export function resolveImagePipelineFromKb( + kb: Record | null | undefined, +): string { + const cfg = kb?.image_processing_config + const stored = normalizeImagePipelineId((cfg?.image_pipeline as string | undefined) || '') + if (stored) return stored + return cfg?.image_attrs_enabled ? IMAGE_PIPELINE_SMARTOCR : IMAGE_PIPELINE_DEFAULT +} diff --git a/frontend/src/views/knowledge/ImagePipelineSettings.test.ts b/frontend/src/views/knowledge/ImagePipelineSettings.test.ts new file mode 100644 index 00000000..06a276da --- /dev/null +++ b/frontend/src/views/knowledge/ImagePipelineSettings.test.ts @@ -0,0 +1,185 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { fileURLToPath } from 'node:url' +import { runInNewContext } from 'node:vm' +import test from 'node:test' +import { compileScript, parse } from '@vue/compiler-sfc' +import ts from 'typescript' +import { createRenderer, h, nextTick, reactive } from 'vue' + +const require = createRequire(import.meta.url) +const filename = fileURLToPath(new URL('./ImagePipelineSettings.vue', import.meta.url)) +const { descriptor } = parse(readFileSync(filename, 'utf8'), { filename }) +const script = compileScript(descriptor, { id: 'image-pipeline-settings-test' }).content + .replace('__expose();', '') + .replace('return __returned__', '__expose(__returned__); return __returned__') +const compiled = ts.transpileModule(script, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }, +}).outputText + +const violation = { message_key: 'imagePipeline.errors.noActionEnabled', field: 'enable_caption' } +const flushUpdates = async () => { + await new Promise((resolve) => setImmediate(resolve)) + await nextTick() +} + +type ValidationRequest = { + pipelineId: string + params: Record + resolve: (value: { violations: typeof violation[] }) => void + reject: (reason: Error) => void +} + +async function fixture() { + const requests: ValidationRequest[] = [] + const timers = new Map void>() + let timerID = 0 + const exports: any = {} + runInNewContext(compiled, { + exports, console: { ...console, error() {} }, + setTimeout(callback: () => void) { timers.set(++timerID, callback); return timerID }, + clearTimeout(id: number) { timers.delete(id) }, + require(name: string) { + if (name === 'vue') return require('vue') + if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key, te: () => false }) } + if (name === '@/api/knowledge-base') return { + fetchImagePipelines: async () => [{ id: 'default', fields: [] }, { id: 'smartocr', fields: [] }], + validateImagePipelines: (pipelineId: string, params: Record) => + new Promise((resolve, reject) => requests.push({ pipelineId, params, resolve, reject })), + } + throw new Error(`Unexpected import: ${name}`) + }, + }) + exports.default.render = () => null + const renderer = createRenderer({ + createElement: () => ({}), createText: () => ({}), createComment: () => ({}), + insert() {}, remove() {}, setElementText() {}, setText() {}, patchProp() {}, + parentNode: () => null, nextSibling: () => null, + }) + const props = reactive({ pipelineId: 'default', params: { enable_caption: false, enable_ocr: false } as Record }) + const emitted: boolean[] = [] + let vm: any + const app = renderer.createApp({ setup: () => () => h(exports.default, { + ...props, + ref: (value: any) => { vm = value }, + 'onUpdate:params': (params: Record) => { props.params = params }, + 'onUpdate:pipelineId': (pipelineId: string) => { props.pipelineId = pipelineId }, + 'onUpdate:invalid': (invalid: boolean) => { emitted.push(invalid) }, + }) }) + app.mount({}) + await flushUpdates() + return { + requests, timers, emitted, + get invalid() { return emitted.at(-1) }, + get validationFailed() { return vm.validationFailed }, + async setCaption(enabled: boolean) { vm.onParamChange('enable_caption', enabled); await flushUpdates() }, + async setPipeline(id: string) { vm.onPipelineChange(id); await flushUpdates() }, + async validate() { + for (const [id, callback] of [...timers]) { timers.delete(id); callback() } + await flushUpdates() + }, + dispose: () => app.unmount(), + } +} + +test('a late invalid response cannot block saving newer valid parameters', async () => { + const f = await fixture() + try { + await f.validate() + await f.setCaption(true) + await f.validate() + assert.equal(f.requests.length, 2) + assert.equal(f.requests[0].params.enable_caption, false) + assert.equal(f.requests[1].params.enable_caption, true) + f.requests[1].resolve({ violations: [] }) + await flushUpdates() + f.requests[0].resolve({ violations: [violation] }) + await flushUpdates() + assert.equal(f.invalid, false) + } finally { f.dispose() } +}) + +for (const response of ['valid', 'error'] as const) { + test(`a stale ${response} response cannot erase the current invalid verdict`, async () => { + const f = await fixture() + try { + await f.setCaption(true) + await f.validate() + await f.setCaption(false) + await f.validate() + f.requests[1].resolve({ violations: [violation] }) + await flushUpdates() + assert.equal(f.invalid, true) + if (response === 'valid') f.requests[0].resolve({ violations: [] }) + else f.requests[0].reject(new Error('old request failed')) + await flushUpdates() + assert.equal(f.invalid, true) + assert.equal(f.validationFailed, false) + } finally { f.dispose() } + }) +} + +test('parameter changes invalidate responses even before the next debounce fires', async () => { + const f = await fixture() + try { + await f.validate() + await f.setCaption(true) + assert.equal(f.requests.length, 1) + f.requests[0].resolve({ violations: [violation] }) + await flushUpdates() + assert.equal(f.invalid, false) + } finally { f.dispose() } +}) + +test('switching pipelines clears the previous verdict and ignores its pending request', async () => { + const f = await fixture() + try { + await f.validate() + f.requests[0].resolve({ violations: [violation] }) + await flushUpdates() + assert.equal(f.invalid, true) + await f.setCaption(true) + await f.validate() + await f.setPipeline('smartocr') + assert.equal(f.invalid, false) + await f.validate() + assert.equal(f.requests[2].pipelineId, 'smartocr') + assert.deepEqual({ ...f.requests[2].params }, {}) + f.requests[2].resolve({ violations: [] }) + f.requests[1].resolve({ violations: [violation] }) + await flushUpdates() + assert.equal(f.invalid, false) + } finally { f.dispose() } +}) + +test('unmount cancels scheduled validation and ignores in-flight results', async () => { + const f = await fixture() + await f.validate() + await f.setCaption(true) + const emitted = [...f.emitted] + f.dispose() + assert.equal(f.timers.size, 0) + f.requests[0].resolve({ violations: [violation] }) + await flushUpdates() + assert.deepEqual(f.emitted, emitted) + await f.validate() + assert.equal(f.requests.length, 1) +}) + +test('a current validation failure still permits saving and is cleared by the next result', async () => { + const f = await fixture() + try { + await f.validate() + f.requests[0].reject(new Error('network unavailable')) + await flushUpdates() + assert.equal(f.invalid, false) + assert.equal(f.validationFailed, true) + await f.setCaption(true) + await f.validate() + f.requests[1].resolve({ violations: [] }) + await flushUpdates() + assert.equal(f.validationFailed, false) + assert.equal(f.invalid, false) + } finally { f.dispose() } +}) diff --git a/frontend/src/views/knowledge/ImagePipelineSettings.vue b/frontend/src/views/knowledge/ImagePipelineSettings.vue new file mode 100644 index 00000000..43804fda --- /dev/null +++ b/frontend/src/views/knowledge/ImagePipelineSettings.vue @@ -0,0 +1,334 @@ + + + + + diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index eb5073c3..70ea3a36 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -402,12 +402,23 @@ -
+
- -

{{ $t('knowledgeEditor.advanced.multimodal.imageAttrsDescription') }}

+ +

+ {{ $t('knowledgeEditor.advanced.multimodal.imagePipelineSectionDescription') }} +

- + +
{ descriptionLanguage: '', customInstructions: '' }, - // 图片属性观察管线:开关 + on_unobserved 兜底切换由 UI 编辑;其余配置 - // (model_id 等)按加载时的快照原样回传,避免把 API 侧写入的设置洗掉。 - // 新建模式也必须用完整默认动作初始化——imageActions.ocr.on_unobserved - // 直接被开关绑定,缺省会让打开开关的瞬间渲染崩溃。 + // 图片解析方案:方案选择与各方案的私有参数由 ImagePipelineSettings + // 编辑;其余配置(model_id 等)按加载时的快照原样回传,避免把 API 侧写入 + // 的设置洗掉。新建模式也必须用完整默认动作初始化——imageActions.ocr + // .on_unobserved 直接被开关绑定,缺省会让打开开关的瞬间渲染崩溃。 imageAttrsEnabled: false, // 图片向量默认关闭:能处理图片的向量模型也常只用来编码文本。 imageVectorEnabled: false, imageActions: mergeImageActions(), + // 方案没有「留空」选项:多模态开关决定是否处理图片,开了就一定要有 + // 方案可跑,所以默认落在 default(内部名,UI 显示为「传统」)。选中的 + // 方案决定旧观察开关的取值(见 onPipelineChange)。 + imagePipeline: IMAGE_PIPELINE_DEFAULT, + imagePipelineParams: {} as Record, imageProcessingConfigSnapshot: null as Record | null, asrConfig: { enabled: false, @@ -1107,13 +1129,24 @@ const loadKBData = async ( descriptionLanguage: kb.vlm_config?.description_language || '', customInstructions: kb.vlm_config?.custom_instructions || '' }, - imageAttrsEnabled: - !!(kb as Record).image_processing_config?.image_attrs_enabled, + // 旧库迁移:本选择器出现之前保存的知识库没有 image_pipeline,按后端 + // ResolveImagePipelineID 的同一规则落到具体方案——观察开关开着就是 + // smartocr,其余(包括根本没有 image_processing_config 的老库)落到 + // default。存过的 id 还可能是更名前的 caption_ocr / ob_cap_ocr, + // 归一化规则在 resolveKbImagePipeline 里,与上传/重新解析弹窗共用。 + imagePipeline: resolveKbImagePipeline(kb), + // 旧观察开关与选中的方案保持同一个意思:只有智能模式把它点亮。 + imageAttrsEnabled: resolveKbImagePipeline(kb) === IMAGE_PIPELINE_SMARTOCR, imageVectorEnabled: !!(kb as Record).image_processing_config?.image_vector_enabled, imageActions: mergeImageActions( (kb as Record).image_processing_config?.image_actions, ), + imagePipelineParams: + ((kb as Record).image_processing_config?.image_pipeline_params as Record< + string, + unknown + >) ?? {}, imageProcessingConfigSnapshot: (kb as Record).image_processing_config || null, asrConfig: { @@ -1295,6 +1328,29 @@ const handleMultimodalToggle = () => { } } +/** + * 知识库存量配置到方案 id 的解析规则已抽到 utils/imageProcessingConfig + * (与后端 ResolveImagePipelineID 一致),供本弹窗与上传/重解析弹窗共用。 + */ + +/** + * 流水线参数的清理由 ImagePipelineSettings 自己完成(切换即清空上一条的 + * 私有参数);这里只负责让旧的观察开关跟上选择——只有观察属性流水线把它 + * 点亮,这样无论从哪个控件看,知识库读到的是同一个意思,下方的属性面板 + * 也跟随这个开关显隐。 + */ +const onPipelineChange = (value: string) => { + if (!formData.value) return + formData.value.imagePipeline = value + formData.value.imageAttrsEnabled = value === IMAGE_PIPELINE_SMARTOCR +} + +// 面板报上来的「选中的方案一个动作都没开」。只在多模态开启时阻止保存。 +const imagePipelineInvalid = ref(false) +const onPipelineInvalidChange = (value: boolean) => { + imagePipelineInvalid.value = value +} + const handleMultimodalVLLMChange = (modelId: string) => { if (formData.value) { formData.value.multimodalConfig.vllmModelId = modelId @@ -1441,6 +1497,15 @@ const validateForm = (): boolean => { return false } + // 选中的解析方案违反了它自己的规则(如传统方案两个动作开关都关):跑起来 + // 什么都不会做,属于配置错误而不是合法选择。提示文案与面板底部那几行同源, + // 因此规则加得多、提示也就跟着变,这里不必跟着改。 + if (formData.value.multimodalConfig.enabled && imagePipelineInvalid.value) { + MessagePlugin.warning(t('imagePipeline.errors.noActionEnabled')) + currentSection.value = 'multimodal' + return false + } + if (formData.value.type === 'faq' && !formData.value.faqConfig?.indexMode) { MessagePlugin.warning(t('knowledgeEditor.messages.indexModeRequired')) currentSection.value = 'faq' @@ -1508,6 +1573,10 @@ const buildSubmitData = () => { imageVectorEnabled: formData.value.imageVectorEnabled, onUnobserved: formData.value.imageActions.ocr.on_unobserved, defaultOn: displaySchema.value.default_actions.ocr.on, + // 多模态开关决定是否处理图片:关闭时不写流水线字段(buildImageProcessingConfig + // 对空 pipelineId 的处理是删掉这两个字段),配置里只留下观察开关的语义。 + pipelineId: formData.value.multimodalConfig.enabled ? formData.value.imagePipeline : '', + pipelineParams: formData.value.imagePipelineParams, }) if (built) { data.image_processing_config = built diff --git a/frontend/src/views/knowledge/components/UploadConfirmDialog.vue b/frontend/src/views/knowledge/components/UploadConfirmDialog.vue index ba04d3b3..7d6a4b10 100644 --- a/frontend/src/views/knowledge/components/UploadConfirmDialog.vue +++ b/frontend/src/views/knowledge/components/UploadConfirmDialog.vue @@ -421,7 +421,7 @@ />
-
+

{{ t('uploadConfirm.imageIndexInherited') }}

@@ -450,15 +450,22 @@
-
+
- -

{{ t('knowledgeEditor.advanced.multimodal.imageAttrsDescription') }}

+ +

{{ t('knowledgeEditor.advanced.multimodal.imagePipelineSectionDescription') }}

- + +
@@ -504,7 +511,7 @@ .setting-info / .setting-control 都撑成 100% 宽,水平 排布必然溢出,开关被顶到容器右缘之外。挪出来后与其它 开关行共用同一套排版,右缘与「图片属性观察」对齐 --> -