diff --git a/internal/models/embedding/image_wire_test.go b/internal/models/embedding/image_wire_test.go index 48d2e6b3..62444145 100644 --- a/internal/models/embedding/image_wire_test.go +++ b/internal/models/embedding/image_wire_test.go @@ -154,6 +154,8 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) { {"text model on the OpenAI shape", "openai", "text-embedding-3-small", nil, false}, {"generic endpoint, image input not declared", "generic", "my-clip", nil, false}, {"declared on a generic (vLLM) endpoint", "generic", "my-clip", declare, true}, + {"declared on the OpenAI provider", "openai", "my-clip", declare, true}, + {"declared on LiteLLM", "litellm", "my-clip", declare, true}, { "declared, but the row switched to an object format without a field", "generic", "my-clip", &types.ModelSpecOverride{ @@ -188,6 +190,35 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) { // image both pass the chat template and the prompt ends in <|endoftext|>, // the token its last-token pooling reads. A plain `input` string skips the // template and lands in a different space from the images. +// An OpenAI-compatible gateway has no image field of its own. Declaring +// image input, and nothing else, must still send the picture: vLLM takes it +// as a messages conversation. The text request stays on input. +func TestDeclaredImageInputOnOpenAIGatewayUsesMessages(t *testing.T) { + up := newUpstream(t) + declare := &types.ModelSpecOverride{Input: []string{"text", "image"}} + e := imageEmbedder(t, up, "openai", "WeMM-Embedding-2B", "/v1", declare) + ie, ok := AsImageEmbedder(e) + require.True(t, ok) + + _, err := e.BatchEmbed(context.Background(), []string{"bridge"}) + require.NoError(t, err) + images, err := ie.BatchEmbedImages(context.Background(), pngs(1)) + require.NoError(t, err) + assert.Equal(t, [][]float32{{1}}, images) + + require.Len(t, up.requests, 2) + assert.Equal(t, map[string]any{ + "model": "WeMM-Embedding-2B", "encoding_format": "float", + "input": []any{"bridge"}, + }, up.requests[0].body) + assert.Equal(t, map[string]any{ + "model": "WeMM-Embedding-2B", "encoding_format": "float", + "messages": []any{map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "image_url", "image_url": map[string]any{"url": pngs(1)[0].DataURI()}}, + }}}, + }, up.requests[1].body) +} + func TestSelfHostedQwen3VLEmbeddingSendsTextAndImagesThroughTheTemplate(t *testing.T) { up := newUpstream(t) e := imageEmbedder(t, up, "generic", "Qwen/Qwen3-VL-Embedding-8B", "/v1", nil) diff --git a/internal/models/runtime/resolve.go b/internal/models/runtime/resolve.go index 0a721d06..04534736 100644 --- a/internal/models/runtime/resolve.go +++ b/internal/models/runtime/resolve.go @@ -492,6 +492,17 @@ func resolveEmbeddings( "catalog: unknown embedding image_format %q on %s/%s (expected %q or %q)", settings.ImageFormat, vendor.ID, spec.ID, api.EmbeddingImageObject, api.EmbeddingImageMessages) } + // A row that declares image input on the OpenAI embedding shape still + // needs a wire format: that shape has none of its own, and only Jina + // names one. vLLM, the usual server behind these gateways, takes one + // chat conversation in messages. Jina and SGLang stay explicit + // (image_format=object plus image_field). An object format without a + // field is left alone, so a half-filled row is refused instead of resent + // as messages. Text stays on input unless the row sets text_as_messages. + if protocol == api.EmbeddingOpenAI && spec.AcceptsImages() && + settings.ImageFormat == "" && settings.ImageField == "" { + settings.ImageFormat = api.EmbeddingImageMessages + } out := &Resolved{ Vendor: configcopy.Clone(vendor.Definition), diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.md index 89633da8..ef1e4a81 100644 --- a/website-docs/03-features/06-models.md +++ b/website-docs/03-features/06-models.md @@ -89,7 +89,7 @@ | 需要附带厂商私有参数,如阿里云联网搜索 | `{"extra_body": {"enable_search": true}}` | | 多轮对话回传思考内容时报错 | `{"replay_reasoning_content": false}` | | 自建向量服务一次请求只接受 16 条 | `{"max_batch_size": 16}` | -| vLLM 部署的多模态向量模型(VLM2Vec、GME 等) | 打开「图片输入」即可,自定义厂商默认按 vLLM 的 `messages` 格式发图片;模板需要指令时加 `{"image_prompt": "Represent the given image."}`;文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` | +| vLLM 部署的多模态向量模型(VLM2Vec、GME 等) | 打开「图片输入」即可。未填 `image_format` 和 `image_field` 时,OpenAI 兼容协议按 vLLM 的 `messages` 格式发图片。模板需要指令时加 `{"image_prompt": "Represent the given image."}`;文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` | | vLLM 部署的 Qwen3-VL-Embedding | 无需填写:自定义厂商下模型名含 `qwen3-vl-embedding` 时已内置图片输入与 `{"text_as_messages": true, "extra_body": {"add_special_tokens": true}}`。文本走普通 `input` 会绕过对话模板,与图片不在同一空间;不加 `add_special_tokens` 时末尾缺少模型取向量用的 `<\|endoftext\|>`,与[模型卡](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)结果不符。已用该模型建立的索引需重建向量 | | SGLang 部署的多模态向量模型 | 打开「图片输入」,并填 `{"image_format": "object", "image_field": "image"}` | | 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` | @@ -235,8 +235,8 @@ WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_toke | `accepts_truncate_prompt_tokens` | bool | false | 服务是否支持 vLLM 的 `truncate_prompt_tokens` | | `request_timeout_seconds` | int | 60 | 单次请求超时(秒) | | `extra_body` | object | 空 | 附加到每次请求的字段 | -| `image_field` | string | 空 | 仅 `openai-embeddings`:`input` 里承载图片的对象键名(Jina、SGLang 为 `image`)。留空且 `image_format` 不是 `messages` 时表示该端点不收图片 | -| `image_format` | string | 空(自定义厂商为 `messages`) | 仅 `openai-embeddings`:图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`;`messages` 把图片作为一轮对话放进 `messages`(vLLM 的多模态向量接口),每张图一个请求。留空时有 `image_field` 即按 `object` 处理 | +| `image_field` | string | 空 | 仅 `openai-embeddings`:`input` 里承载图片的对象键名(Jina、SGLang 为 `image`)。模型未声明图片输入、留空、且 `image_format` 不是 `messages` 时,端点不收图片 | +| `image_format` | string | 空 | 仅 `openai-embeddings`:图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`;`messages` 把图片作为一轮对话放进 `messages`(vLLM 的多模态向量接口)。模型声明了图片输入、本字段和 `image_field` 都留空时,按 `messages` 发送。只填了 `image_field` 时按 `object` 处理。写成 `object` 却没有 `image_field` 时仍不发图片 | | `image_prompt` | string | 空 | 仅 `messages` 格式:随图片一起发送的文字,部分模型的对话模板需要(如 VLM2Vec 的 `Represent the given image.`) | | `text_as_messages` | bool | false | 仅 `openai-embeddings`:文本也按 `messages` 格式发送,每条一个请求。适用于必须经过对话模板、文本和图片才在同一向量空间的模型(自定义厂商下的 Qwen3-VL-Embedding 已内置);已有索引的模型改动后需要重建向量 | | `max_image_batch_size` | int | 1 | 单次请求的最大图片数,超出自动分批 |