fix(embed): send images when an OpenAI-compatible row declares them (#4389)

An embedding row with spec.input image and no image_format or image_field
was treated as text-only on every vendor except generic and Jina. Default
that case to the vLLM messages format. An explicit object format without a
field stays refused.

Fixes #4343
main
JSJ189 2026-10-16 20:15:32 +08:00 committed by GitHub
parent 7ba8e2eba9
commit 7705f77be9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 45 additions and 3 deletions

View File

@ -154,6 +154,8 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) {
{"text model on the OpenAI shape", "openai", "text-embedding-3-small", nil, false},
{"generic endpoint, image input not declared", "generic", "my-clip", nil, false},
{"declared on a generic (vLLM) endpoint", "generic", "my-clip", declare, true},
{"declared on the OpenAI provider", "openai", "my-clip", declare, true},
{"declared on LiteLLM", "litellm", "my-clip", declare, true},
{
"declared, but the row switched to an object format without a field", "generic", "my-clip",
&types.ModelSpecOverride{
@ -188,6 +190,35 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) {
// image both pass the chat template and the prompt ends in <|endoftext|>,
// the token its last-token pooling reads. A plain `input` string skips the
// template and lands in a different space from the images.
// An OpenAI-compatible gateway has no image field of its own. Declaring
// image input, and nothing else, must still send the picture: vLLM takes it
// as a messages conversation. The text request stays on input.
func TestDeclaredImageInputOnOpenAIGatewayUsesMessages(t *testing.T) {
up := newUpstream(t)
declare := &types.ModelSpecOverride{Input: []string{"text", "image"}}
e := imageEmbedder(t, up, "openai", "WeMM-Embedding-2B", "/v1", declare)
ie, ok := AsImageEmbedder(e)
require.True(t, ok)
_, err := e.BatchEmbed(context.Background(), []string{"bridge"})
require.NoError(t, err)
images, err := ie.BatchEmbedImages(context.Background(), pngs(1))
require.NoError(t, err)
assert.Equal(t, [][]float32{{1}}, images)
require.Len(t, up.requests, 2)
assert.Equal(t, map[string]any{
"model": "WeMM-Embedding-2B", "encoding_format": "float",
"input": []any{"bridge"},
}, up.requests[0].body)
assert.Equal(t, map[string]any{
"model": "WeMM-Embedding-2B", "encoding_format": "float",
"messages": []any{map[string]any{"role": "user", "content": []any{
map[string]any{"type": "image_url", "image_url": map[string]any{"url": pngs(1)[0].DataURI()}},
}}},
}, up.requests[1].body)
}
func TestSelfHostedQwen3VLEmbeddingSendsTextAndImagesThroughTheTemplate(t *testing.T) {
up := newUpstream(t)
e := imageEmbedder(t, up, "generic", "Qwen/Qwen3-VL-Embedding-8B", "/v1", nil)

View File

@ -492,6 +492,17 @@ func resolveEmbeddings(
"catalog: unknown embedding image_format %q on %s/%s (expected %q or %q)",
settings.ImageFormat, vendor.ID, spec.ID, api.EmbeddingImageObject, api.EmbeddingImageMessages)
}
// A row that declares image input on the OpenAI embedding shape still
// needs a wire format: that shape has none of its own, and only Jina
// names one. vLLM, the usual server behind these gateways, takes one
// chat conversation in messages. Jina and SGLang stay explicit
// (image_format=object plus image_field). An object format without a
// field is left alone, so a half-filled row is refused instead of resent
// as messages. Text stays on input unless the row sets text_as_messages.
if protocol == api.EmbeddingOpenAI && spec.AcceptsImages() &&
settings.ImageFormat == "" && settings.ImageField == "" {
settings.ImageFormat = api.EmbeddingImageMessages
}
out := &Resolved{
Vendor: configcopy.Clone(vendor.Definition),

View File

@ -89,7 +89,7 @@
| 需要附带厂商私有参数,如阿里云联网搜索 | `{"extra_body": {"enable_search": true}}` |
| 多轮对话回传思考内容时报错 | `{"replay_reasoning_content": false}` |
| 自建向量服务一次请求只接受 16 条 | `{"max_batch_size": 16}` |
| vLLM 部署的多模态向量模型(VLM2Vec、GME 等) | 打开「图片输入」即可,自定义厂商默认按 vLLM 的 `messages` 格式发图片;模板需要指令时加 `{"image_prompt": "Represent the given image."}`;文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` |
| vLLM 部署的多模态向量模型(VLM2Vec、GME 等) | 打开「图片输入」即可。未填 `image_format` 和 `image_field` 时,OpenAI 兼容协议按 vLLM 的 `messages` 格式发图片。模板需要指令时加 `{"image_prompt": "Represent the given image."}`;文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` |
| vLLM 部署的 Qwen3-VL-Embedding | 无需填写:自定义厂商下模型名含 `qwen3-vl-embedding` 时已内置图片输入与 `{"text_as_messages": true, "extra_body": {"add_special_tokens": true}}`。文本走普通 `input` 会绕过对话模板,与图片不在同一空间;不加 `add_special_tokens` 时末尾缺少模型取向量用的 `<\|endoftext\|>`,与[模型卡](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)结果不符。已用该模型建立的索引需重建向量 |
| SGLang 部署的多模态向量模型 | 打开「图片输入」,并填 `{"image_format": "object", "image_field": "image"}` |
| 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` |
@ -235,8 +235,8 @@ WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_toke
| `accepts_truncate_prompt_tokens` | bool | false | 服务是否支持 vLLM 的 `truncate_prompt_tokens` |
| `request_timeout_seconds` | int | 60 | 单次请求超时(秒) |
| `extra_body` | object | 空 | 附加到每次请求的字段 |
| `image_field` | string | 空 | 仅 `openai-embeddings`:`input` 里承载图片的对象键名(Jina、SGLang 为 `image`)。留空且 `image_format` 不是 `messages` 时表示该端点不收图片 |
| `image_format` | string | 空(自定义厂商为 `messages`) | 仅 `openai-embeddings`:图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`;`messages` 把图片作为一轮对话放进 `messages`(vLLM 的多模态向量接口),每张图一个请求。留空时有 `image_field` 即按 `object` 处理 |
| `image_field` | string | 空 | 仅 `openai-embeddings`:`input` 里承载图片的对象键名(Jina、SGLang 为 `image`)。模型未声明图片输入、留空、且 `image_format` 不是 `messages` 时,端点不收图片 |
| `image_format` | string | 空 | 仅 `openai-embeddings`:图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`;`messages` 把图片作为一轮对话放进 `messages`(vLLM 的多模态向量接口)。模型声明了图片输入、本字段和 `image_field` 都留空时,按 `messages` 发送。只填了 `image_field` 时按 `object` 处理。写成 `object` 却没有 `image_field` 时仍不发图片 |
| `image_prompt` | string | 空 | 仅 `messages` 格式:随图片一起发送的文字,部分模型的对话模板需要(如 VLM2Vec 的 `Represent the given image.`) |
| `text_as_messages` | bool | false | 仅 `openai-embeddings`:文本也按 `messages` 格式发送,每条一个请求。适用于必须经过对话模板、文本和图片才在同一向量空间的模型(自定义厂商下的 Qwen3-VL-Embedding 已内置);已有索引的模型改动后需要重建向量 |
| `max_image_batch_size` | int | 1 | 单次请求的最大图片数,超出自动分批 |