From c3f077ed18f6e00aaf705d405023ed88c7c15b27 Mon Sep 17 00:00:00 2001 From: lyingbug Date: Sat, 10 Oct 2026 11:06:00 +0800 Subject: [PATCH] fix(models): align configuration and multimodal embedding workflows (#4080) * fix(models): align configuration defaults and multimodal workflows * feat(image): enhance image retrieval diagnostics and filtering - Implemented `retrieveObservedImagePool` to observe engine calls and record image recall decisions without altering results. - Added detailed diagnostics in `summarizeImagePool` and `summarizeImageCandidateCut` to track image filtering decisions and scores. - Introduced tests for image pool diagnostics to ensure accurate filtering and decision recording. - Updated existing functions to utilize new retrieval and summarization methods for improved observability and accuracy in image handling. * feat(embeddings): enhance image handling with new formats and message support - Introduced `EmbeddingImageFormat` to manage image input formats, allowing for both object and messages formats. - Updated `EmbeddingsSettings` to include new fields for `ImageFormat`, `ImagePrompt`, and `TextAsMessages` to support multimodal embedding workflows. - Enhanced `Client` methods to build request bodies for images in both formats, ensuring compatibility with vLLM's requirements. - Added tests to validate the new image message handling and ensure correct request formatting for different scenarios. - Updated documentation to reflect changes in image input handling and configuration options. * feat(similarity): add embedding similarity mode for enhanced document scoring - Introduced a new `similarity` option in `ModelDebugOptions` to score documents and an optional image against a query. - Updated `ModelDebugDrawer.vue` to support a new input mode for similarity, allowing users to select similarity as an input type. - Enhanced the UI to display similarity results, including ranked candidates based on cosine similarity. - Added tests to validate the functionality of the new similarity mode in embedding models. - Updated localization files to include new strings related to similarity mode in multiple languages. * feat(embeddings): add Qwen3-VL-Embedding support for multimodal input - Introduced Qwen3-VL-Embedding model with support for both text and image inputs. - Updated model configuration files to include compatibility settings for the new embedding type. - Enhanced tests to validate the functionality of Qwen3-VL-Embedding, ensuring correct handling of inputs and outputs. - Updated documentation to reflect the new model's capabilities and usage instructions. * test(models): wrap embedding assertions to satisfy lint * test(opensearch): isolate HTTP transports between parallel tests --- frontend/src/api/model/index.ts | 2 + frontend/src/components/ModelDebugDrawer.vue | 195 +++++++-- frontend/src/components/ModelEditorDialog.vue | 114 +++--- .../src/components/modelDebugImages.test.ts | 177 ++++++++ .../src/components/modelEditorFlow.test.ts | 236 ++++++++++- frontend/src/i18n/locales/en-US.ts | 13 + frontend/src/i18n/locales/ja-JP.ts | 13 + frontend/src/i18n/locales/ko-KR.ts | 13 + frontend/src/i18n/locales/ru-RU.ts | 13 + frontend/src/i18n/locales/zh-CN.ts | 13 + frontend/src/i18n/locales/zh-TW.ts | 13 + frontend/src/types/knowledgeProcess.ts | 2 + .../components/UploadConfirmDialog.vue | 39 ++ .../uploadImageVectorOverride.test.ts | 90 +++++ .../retriever/opensearch/repository_test.go | 32 ++ .../application/service/image_multimodal.go | 14 +- .../service/image_multimodal_vector_test.go | 63 +++ .../application/service/knowledge_process.go | 29 +- .../service/knowledge_process_config.go | 4 + .../service/knowledgebase_search.go | 2 + .../service/knowledgebase_search_fanout.go | 9 +- .../service/knowledgebase_search_image.go | 44 +- .../service/knowledgebase_search_image_obs.go | 140 +++++++ .../knowledgebase_search_image_obs_test.go | 65 +++ .../knowledgebase_search_image_test.go | 4 +- internal/handler/dto/model.go | 7 +- internal/handler/model.go | 102 ++++- internal/handler/model_catalog.go | 3 + .../handler/model_catalog_visibility_test.go | 64 +++ internal/handler/model_debug_images_test.go | 382 ++++++++++++++++++ internal/handler/model_debug_similarity.go | 110 +++++ .../anthropicmessages_test.go | 1 + internal/models/api/embeddings_settings.go | 61 ++- .../models/api/openaiembeddings/client.go | 71 +++- .../api/openaiembeddings/golden_test.go | 88 ++++ .../models/catalog/data/models.generated.json | 91 +++++ internal/models/catalog/data/overrides.json | 53 +++ internal/models/catalog/data/seed.json | 58 +++ internal/models/embedding/image_wire_test.go | 60 ++- internal/models/embedding/wire_test.go | 4 + .../parity/testdata/catalog-baseline.json | 24 +- internal/models/providers/generic.go | 5 + internal/models/providers/minimax.go | 20 +- internal/models/providers/providers_test.go | 59 +++ internal/models/runtime/resolve.go | 5 + internal/reranking/image.go | 2 +- internal/reranking/observability.go | 115 ++++++ internal/reranking/observability_test.go | 55 +++ internal/reranking/rerank.go | 3 + internal/tracing/langfuse/retrieval_obs.go | 7 + .../tracing/langfuse/retrieval_obs_test.go | 18 + internal/types/knowledge_process.go | 4 + internal/types/task.go | 3 + website-docs/03-features/06-models.md | 8 +- 54 files changed, 2631 insertions(+), 191 deletions(-) create mode 100644 frontend/src/components/modelDebugImages.test.ts create mode 100644 frontend/src/views/knowledge/components/uploadImageVectorOverride.test.ts create mode 100644 internal/application/service/knowledgebase_search_image_obs.go create mode 100644 internal/application/service/knowledgebase_search_image_obs_test.go create mode 100644 internal/handler/model_debug_images_test.go create mode 100644 internal/handler/model_debug_similarity.go create mode 100644 internal/reranking/observability.go create mode 100644 internal/reranking/observability_test.go diff --git a/frontend/src/api/model/index.ts b/frontend/src/api/model/index.ts index 97c4687b..69f6b1c4 100644 --- a/frontend/src/api/model/index.ts +++ b/frontend/src/api/model/index.ts @@ -235,6 +235,8 @@ export interface ModelDebugOptions { thinking?: boolean // Graded thinking level; takes precedence over the boolean when set. reasoning_effort?: ReasoningEffortLevel | string + // Embedding only: score `documents` (and the file) against `input` as a query. + similarity?: boolean } export interface ModelDebugResult { diff --git a/frontend/src/components/ModelDebugDrawer.vue b/frontend/src/components/ModelDebugDrawer.vue index dd734edc..56eff9b5 100644 --- a/frontend/src/components/ModelDebugDrawer.vue +++ b/frontend/src/components/ModelDebugDrawer.vue @@ -76,33 +76,40 @@