feat(models): add WeMM embedding presets and Transformers serving (#4238)
parent
8af7938aed
commit
94fae6de43
|
|
@ -2185,6 +2185,75 @@
|
|||
"add_special_tokens": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-2B",
|
||||
"match": "*/WeMM-Embedding-2B",
|
||||
"type": "Embedding",
|
||||
"name": "WeMM-Embedding-2B",
|
||||
"aliases": [
|
||||
"WeMM-Embedding-2B"
|
||||
],
|
||||
"input": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"dimension": 2048,
|
||||
"source": "https://huggingface.co/tencent/WeMM-Embedding-2B",
|
||||
"compat": {
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_image_batch_size": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-4B",
|
||||
"match": "*/WeMM-Embedding-4B",
|
||||
"type": "Embedding",
|
||||
"name": "WeMM-Embedding-4B",
|
||||
"aliases": [
|
||||
"WeMM-Embedding-4B"
|
||||
],
|
||||
"input": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"dimension": 2560,
|
||||
"source": "https://huggingface.co/tencent/WeMM-Embedding-4B",
|
||||
"compat": {
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_image_batch_size": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-9B",
|
||||
"match": "*/WeMM-Embedding-9B",
|
||||
"type": "Embedding",
|
||||
"name": "WeMM-Embedding-9B",
|
||||
"aliases": [
|
||||
"WeMM-Embedding-9B"
|
||||
],
|
||||
"input": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"dimension": 4096,
|
||||
"source": "https://huggingface.co/tencent/WeMM-Embedding-9B",
|
||||
"compat": {
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_image_batch_size": 5
|
||||
}
|
||||
}
|
||||
],
|
||||
"gpustack": [],
|
||||
|
|
|
|||
|
|
@ -1050,6 +1050,45 @@
|
|||
"add_special_tokens": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-2B",
|
||||
"match": "*/WeMM-Embedding-2B",
|
||||
"type": "Embedding",
|
||||
"compat": {
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_image_batch_size": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-4B",
|
||||
"match": "*/WeMM-Embedding-4B",
|
||||
"type": "Embedding",
|
||||
"compat": {
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_image_batch_size": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-9B",
|
||||
"match": "*/WeMM-Embedding-9B",
|
||||
"type": "Embedding",
|
||||
"compat": {
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_image_batch_size": 5
|
||||
}
|
||||
}
|
||||
],
|
||||
"hunyuan": [
|
||||
|
|
|
|||
|
|
@ -1454,6 +1454,51 @@
|
|||
"image"
|
||||
],
|
||||
"source": "https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B"
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-2B",
|
||||
"match": "*/WeMM-Embedding-2B",
|
||||
"type": "Embedding",
|
||||
"name": "WeMM-Embedding-2B",
|
||||
"aliases": [
|
||||
"WeMM-Embedding-2B"
|
||||
],
|
||||
"input": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"dimension": 2048,
|
||||
"source": "https://huggingface.co/tencent/WeMM-Embedding-2B"
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-4B",
|
||||
"match": "*/WeMM-Embedding-4B",
|
||||
"type": "Embedding",
|
||||
"name": "WeMM-Embedding-4B",
|
||||
"aliases": [
|
||||
"WeMM-Embedding-4B"
|
||||
],
|
||||
"input": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"dimension": 2560,
|
||||
"source": "https://huggingface.co/tencent/WeMM-Embedding-4B"
|
||||
},
|
||||
{
|
||||
"id": "tencent/WeMM-Embedding-9B",
|
||||
"match": "*/WeMM-Embedding-9B",
|
||||
"type": "Embedding",
|
||||
"name": "WeMM-Embedding-9B",
|
||||
"aliases": [
|
||||
"WeMM-Embedding-9B"
|
||||
],
|
||||
"input": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"dimension": 4096,
|
||||
"source": "https://huggingface.co/tencent/WeMM-Embedding-9B"
|
||||
}
|
||||
],
|
||||
"gpustack": [],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
modelruntime "github.com/Tencent/WeKnora/internal/models/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// WeMM's vLLM template appends the dedicated <embedding> pooling token.
|
||||
// Both retrieval sides and images must use that template, including when
|
||||
// vLLM exposes the local model path instead of the Hugging Face repository ID.
|
||||
// https://huggingface.co/tencent/WeMM-Embedding-2B/blob/main/embedding_chat_template.jinja
|
||||
func TestWeMMVLLMPresets(t *testing.T) {
|
||||
vendor, ok := modelruntime.Get("generic")
|
||||
require.True(t, ok)
|
||||
listed := vendor.ModelsByType(types.ModelTypeEmbedding)
|
||||
for _, tc := range []struct {
|
||||
size string
|
||||
dims int
|
||||
}{{"2B", 2048}, {"4B", 2560}, {"9B", 4096}} {
|
||||
short := "WeMM-Embedding-" + tc.size
|
||||
id := "tencent/" + short
|
||||
t.Run(tc.size, func(t *testing.T) {
|
||||
visible := false
|
||||
for _, entry := range listed {
|
||||
if entry.ID == id {
|
||||
visible = true
|
||||
require.Equal(t, tc.dims, entry.Dimension)
|
||||
}
|
||||
}
|
||||
require.True(t, visible, "preset must appear in the embedding picker")
|
||||
for _, name := range []string{id, short, "/models/" + short, strings.ToLower(id)} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
resolved, err := modelruntime.Resolve(modelruntime.Ref{
|
||||
Provider: "generic", Model: name, ModelType: types.ModelTypeEmbedding,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resolved.Cataloged)
|
||||
require.Equal(t, tc.dims, resolved.Spec.Dimension)
|
||||
require.Equal(t, api.EmbeddingOpenAI, resolved.EmbeddingAPI)
|
||||
require.Equal(t, []string{"text", "image"}, resolved.Spec.Input)
|
||||
require.True(t, resolved.Embeddings.BatchMessages)
|
||||
require.Equal(t, 5, resolved.Embeddings.ImageBatchLimit())
|
||||
|
||||
up := newUpstream(t)
|
||||
e, err := NewEmbedder(Config{
|
||||
Source: types.ModelSourceRemote, Provider: "generic", ModelName: name,
|
||||
BaseURL: up.url + "/v1", Dimensions: tc.dims, TruncatePromptTokens: 511,
|
||||
}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.dims, e.GetDimensions())
|
||||
ie, ok := AsImageEmbedder(e)
|
||||
require.True(t, ok, "images must work without a row override")
|
||||
contexts := []context.Context{context.Background(), types.WithEmbedQuery(context.Background())}
|
||||
for _, ctx := range contexts {
|
||||
vectors, err := e.BatchEmbed(ctx, []string{"ab", "c"})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, [][]float32{{2}, {1}}, vectors)
|
||||
}
|
||||
vectors, err := ie.BatchEmbedImages(context.Background(), pngs(6))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, [][]float32{{1}, {2}, {3}, {4}, {5}, {6}}, vectors)
|
||||
|
||||
conversation := func(part map[string]any) []any {
|
||||
return []any{map[string]any{"role": "user", "content": []any{part}}}
|
||||
}
|
||||
textBatch := []any{
|
||||
conversation(map[string]any{"type": "text", "text": "ab"}),
|
||||
conversation(map[string]any{"type": "text", "text": "c"}),
|
||||
}
|
||||
imageBatch := []any{}
|
||||
for _, img := range pngs(5) {
|
||||
imageBatch = append(imageBatch, conversation(map[string]any{
|
||||
"type": "image_url", "image_url": map[string]any{"url": img.DataURI()},
|
||||
}))
|
||||
}
|
||||
messages := []any{textBatch, textBatch, imageBatch, conversation(map[string]any{
|
||||
"type": "image_url", "image_url": map[string]any{"url": pngs(6)[5].DataURI()},
|
||||
})}
|
||||
require.Len(t, up.requests, len(messages), "batch texts and split six images into five and one")
|
||||
for i, message := range messages {
|
||||
require.Equal(t, "/v1/embeddings", up.requests[i].path)
|
||||
require.Equal(t, map[string]any{
|
||||
"model": name, "encoding_format": "float",
|
||||
"messages": message,
|
||||
}, up.requests[i].body, "no raw input, legacy truncation, or extra image prompt")
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// vLLM reads matryoshka_dimensions from WeMM's config and truncates before
|
||||
// normalizing. Text and image requests must use the same requested width.
|
||||
func TestWeMMDimensionOverride(t *testing.T) {
|
||||
for _, size := range []string{"2B", "4B", "9B"} {
|
||||
t.Run(size, func(t *testing.T) {
|
||||
up := newUpstream(t)
|
||||
e, err := NewEmbedder(Config{
|
||||
Source: types.ModelSourceRemote, Provider: "generic",
|
||||
ModelName: "tencent/WeMM-Embedding-" + size, BaseURL: up.url + "/v1",
|
||||
Dimensions: 1024, SupportsDimensionOverride: true,
|
||||
}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
_, err = e.Embed(types.WithEmbedQuery(context.Background()), "query")
|
||||
require.NoError(t, err)
|
||||
ie, ok := AsImageEmbedder(e)
|
||||
require.True(t, ok)
|
||||
_, err = ie.BatchEmbedImages(context.Background(), pngs(2))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, up.requests, 2)
|
||||
for _, req := range up.requests {
|
||||
require.Equal(t, float64(1024), req.body["dimensions"])
|
||||
require.NotContains(t, req.body, "truncate_prompt_tokens")
|
||||
require.NotContains(t, req.body, "add_special_tokens")
|
||||
require.NotContains(t, req.body, "add_generation_prompt")
|
||||
require.NotContains(t, req.body, "use_activation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -107,7 +107,16 @@
|
|||
"gemini/KnowledgeQA/gemini-3.7-flash": "570b30391590c3fa380e98e8323bf2187635d18db427eae3d75f9e63def26c8d",
|
||||
"gemini/KnowledgeQA/gemini-3.8-flash": "6d40802ed8d3b310e1e91c186f769a4f8a298138538853cf623ad43e59c29140",
|
||||
"gemini/KnowledgeQA/gemini-3snapshot": "c5931f2304774b718637f47540e78b66cbf855b4a4189c9fc0cea2040a12951d",
|
||||
"generic/Embedding/WeMM-Embedding-2B": "8b6d496d4e145cebe75c192377b9a428be2b049fb23419b07e2f4ed766ee744d",
|
||||
"generic/Embedding/WeMM-Embedding-4B": "d1a6ae0fbde3a9828d6392069aef8707817061b553cfc871b2f5432a460b6e83",
|
||||
"generic/Embedding/WeMM-Embedding-9B": "aa6fce5495ec995031fb677c0396671b79864ab099587ac6cbc3fa52c0868e39",
|
||||
"generic/Embedding/snapshot/WeMM-Embedding-2B": "4832569c35ef95ad39fc75e903fa3a860127c373e06670148301dffbaaf65294",
|
||||
"generic/Embedding/snapshot/WeMM-Embedding-4B": "39c3cfdf00aacdcd443309ae6d4940a0ec16300d52aa1944460feae0174353bd",
|
||||
"generic/Embedding/snapshot/WeMM-Embedding-9B": "ad2407f2bedd74df45f4cc2debe4a8f44be7c9bf138d1228c55924c477ea9a61",
|
||||
"generic/Embedding/snapshotqwen3-vl-embeddingsnapshot": "a1dedaaddb23ad1449d1aef38f0f032d7bcd834bdf079c511c65ec03cccfffe4",
|
||||
"generic/Embedding/tencent/WeMM-Embedding-2B": "54141d358d96ea836ce5a0ede691db0e5dff87cf7d33319a4a59d17a0c1ce0c4",
|
||||
"generic/Embedding/tencent/WeMM-Embedding-4B": "0dd3bbf5caadcd0c230f5bece1aff09e6ccf6180b095c785244580936aeec68b",
|
||||
"generic/Embedding/tencent/WeMM-Embedding-9B": "7176482974a360aa14a1e97dc2e7b508b3ec2526d6f38acbf7f2bf4de595b2d3",
|
||||
"generic/KnowledgeQA/gpt-5snapshot": "3e3712264316db008c1465a2d7c395072d85942d0352196e456342cef0b14eb0",
|
||||
"generic/KnowledgeQA/o1snapshot": "52340886b4160a0cb9342ae33a7c81304c9dad5d22c2d1c1ce073f89860a30b1",
|
||||
"generic/KnowledgeQA/o3snapshot": "782e836769e56c43d9c954ab2d563df7d2f476ae4fb4d588bb8bf3c88ae42ce6",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
# Install torch/torchvision for the target CPU/CUDA/MPS platform separately.
|
||||
# Versions below match the WeMM model card and the validated service environment.
|
||||
transformers==5.2.0
|
||||
qwen-vl-utils==0.0.14
|
||||
accelerate>=1.1.0,<2
|
||||
fastapi>=0.115,<1
|
||||
pydantic>=2,<3
|
||||
uvicorn>=0.30,<1
|
||||
pillow>=10,<13
|
||||
httpx>=0.27,<1
|
||||
|
|
@ -0,0 +1,367 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Serve WeMM text/image embeddings with the official Transformers model code."""
|
||||
import argparse
|
||||
import base64
|
||||
import binascii
|
||||
import hmac
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictInt
|
||||
|
||||
log = logging.getLogger("uvicorn.error")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
model: str = "tencent/WeMM-Embedding-2B"
|
||||
served_model_name: str | None = None
|
||||
revision: str | None = None
|
||||
device: str = "auto"
|
||||
dtype: str = "auto"
|
||||
max_tokens: int = 8192
|
||||
max_pixels: int = 1048576
|
||||
max_batch_size: int = 32
|
||||
cpu_threads: int = 8
|
||||
local_files_only: bool = False
|
||||
api_key: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if min(self.max_tokens, self.max_pixels, self.max_batch_size, self.cpu_threads) <= 0:
|
||||
raise ValueError("Resource limits and cpu_threads must be positive.")
|
||||
if self.max_pixels < 65536:
|
||||
raise ValueError("max_pixels must be at least 65536 (the model processor's minimum).")
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
if self.served_model_name:
|
||||
return self.served_model_name
|
||||
return Path(self.model).name if Path(self.model).is_dir() else self.model
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TextPart(StrictModel):
|
||||
type: Literal["text"]
|
||||
text: str
|
||||
|
||||
|
||||
class ImageURL(StrictModel):
|
||||
url: str = Field(max_length=16 * 1024 * 1024)
|
||||
detail: Literal["auto", "low", "high"] = "auto"
|
||||
|
||||
|
||||
class ImagePart(StrictModel):
|
||||
type: Literal["image_url"]
|
||||
image_url: ImageURL
|
||||
|
||||
|
||||
class Message(StrictModel):
|
||||
role: Literal["system", "user", "assistant"]
|
||||
content: str | list[TextPart | ImagePart] = Field(min_length=1)
|
||||
|
||||
|
||||
class EmbedRequest(StrictModel):
|
||||
model: str
|
||||
input: str | list[str] | None = None
|
||||
messages: list[Message] | list[list[Message]] | None = None
|
||||
dimensions: StrictInt | None = None
|
||||
encoding_format: Literal["float", "base64"] = "float"
|
||||
user: str | None = None
|
||||
|
||||
|
||||
def bad(message):
|
||||
raise HTTPException(400, message)
|
||||
|
||||
|
||||
def parse_inputs(request, settings, dimensions, default_dimension):
|
||||
if request.model != settings.name:
|
||||
raise HTTPException(404, f"Model not found: {request.model}")
|
||||
dimension = default_dimension if request.dimensions is None else request.dimensions
|
||||
if dimension not in dimensions:
|
||||
bad(f"dimensions must be one of {dimensions}.")
|
||||
if (request.input is None) == (request.messages is None):
|
||||
bad("Supply exactly one of input or messages.")
|
||||
if request.input is not None:
|
||||
texts = [request.input] if isinstance(request.input, str) else request.input
|
||||
conversations = [[Message(role="user", content=[TextPart(type="text", text=text)])] for text in texts]
|
||||
else:
|
||||
messages = request.messages
|
||||
conversations = [messages] if messages and isinstance(messages[0], Message) else messages
|
||||
if not conversations or len(conversations) > settings.max_batch_size:
|
||||
bad(f"Supply between 1 and {settings.max_batch_size} independent inputs.")
|
||||
for conversation in conversations:
|
||||
if not conversation:
|
||||
bad("Each conversation must contain at least one message.")
|
||||
chars, images = 0, 0
|
||||
for message in conversation:
|
||||
parts = message.content
|
||||
if isinstance(parts, str):
|
||||
chars += len(parts)
|
||||
continue
|
||||
for part in parts:
|
||||
if isinstance(part, TextPart):
|
||||
chars += len(part.text)
|
||||
else:
|
||||
images += 1
|
||||
if not part.image_url.url.startswith("data:image/") or ";base64," not in part.image_url.url:
|
||||
bad("Images must be base64 data:image/... URLs; network URLs and file paths are unsupported.")
|
||||
if images > 5:
|
||||
bad("At most 5 images per conversation.")
|
||||
if chars > settings.max_tokens * 16:
|
||||
bad("Text exceeds the character guard; reduce the input size.")
|
||||
return conversations, dimension
|
||||
|
||||
|
||||
def decode_image(url):
|
||||
try:
|
||||
data = base64.b64decode(url.split(";base64,", 1)[1], validate=True)
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
bad("Image exceeds 40 megapixels before resizing.")
|
||||
# Preserve alpha until qwen-vl-utils composites RGBA onto white,
|
||||
# matching the official inference path instead of exposing hidden RGB.
|
||||
return image.copy()
|
||||
except (ValueError, binascii.Error, UnidentifiedImageError, OSError, Image.DecompressionBombError):
|
||||
bad("Invalid image data.")
|
||||
|
||||
|
||||
def prepare_messages(conversation, max_pixels):
|
||||
messages = []
|
||||
for message in conversation:
|
||||
parts = message.content
|
||||
if isinstance(parts, str):
|
||||
parts = [TextPart(type="text", text=parts)]
|
||||
content = []
|
||||
for part in parts:
|
||||
if isinstance(part, TextPart):
|
||||
content.append({"type": "text", "text": part.text})
|
||||
else:
|
||||
content.append({"type": "image", "image": decode_image(part.image_url.url), "max_pixels": max_pixels})
|
||||
messages.append({"role": message.role, "content": content})
|
||||
return messages
|
||||
|
||||
|
||||
def resolve_device_dtype(torch, settings):
|
||||
name = settings.device
|
||||
if name == "auto":
|
||||
name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
|
||||
device = torch.device(name)
|
||||
if device.type not in ("cuda", "mps", "cpu"):
|
||||
raise ValueError("device must be auto, cpu, mps, cuda or cuda:N.")
|
||||
if device.type == "cuda":
|
||||
if not torch.cuda.is_available():
|
||||
raise ValueError("CUDA is unavailable; install a CUDA-enabled PyTorch build or select another device.")
|
||||
if device.index is not None and device.index >= torch.cuda.device_count():
|
||||
raise ValueError("The requested CUDA device does not exist.")
|
||||
with torch.cuda.device(device):
|
||||
bf16 = torch.cuda.is_bf16_supported()
|
||||
elif device.type == "mps":
|
||||
if not torch.backends.mps.is_available():
|
||||
raise ValueError("MPS is unavailable on this machine.")
|
||||
bf16 = True
|
||||
else:
|
||||
bf16 = False
|
||||
dtype = settings.dtype
|
||||
if dtype == "auto":
|
||||
dtype = "float32" if device.type == "cpu" else "bfloat16" if bf16 else "float16"
|
||||
if device.type == "cuda" and dtype == "bfloat16" and not bf16:
|
||||
raise ValueError("This CUDA device does not support bfloat16; use --dtype float16.")
|
||||
return device, getattr(torch, dtype)
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(self, settings):
|
||||
# Import after CLI setup so --mps-fallback takes effect before torch loads.
|
||||
import torch
|
||||
from transformers import AutoConfig, AutoModel, AutoProcessor
|
||||
self.settings, self.torch = settings, torch
|
||||
self.device, self.dtype = resolve_device_dtype(torch, settings)
|
||||
self.lock = threading.Lock()
|
||||
torch.set_num_threads(settings.cpu_threads)
|
||||
options = {"trust_remote_code": True, "local_files_only": settings.local_files_only}
|
||||
if settings.revision:
|
||||
options["revision"] = settings.revision
|
||||
config = AutoConfig.from_pretrained(settings.model, **options)
|
||||
self.dimensions = tuple(getattr(config, "matryoshka_dimensions", ()))
|
||||
self.default_dimension = config.text_config.hidden_size
|
||||
if not self.dimensions or self.default_dimension not in self.dimensions:
|
||||
raise ValueError("The model must provide WeMM matryoshka_dimensions including its hidden size.")
|
||||
self.processor = AutoProcessor.from_pretrained(settings.model, **options)
|
||||
self.model = AutoModel.from_pretrained(
|
||||
settings.model, dtype=self.dtype, attn_implementation="sdpa", **options,
|
||||
).to(self.device).eval()
|
||||
if not callable(getattr(self.model, "embedding", None)):
|
||||
raise ValueError("The model must expose WeMM's embedding() method.")
|
||||
|
||||
def encode(self, conversations, dimension):
|
||||
from qwen_vl_utils import process_vision_info
|
||||
torch = self.torch
|
||||
vectors, total_tokens = [], 0
|
||||
# WeMM's model stores rope_deltas. Keep independent conversations serialized,
|
||||
# and decode images only for the current input rather than the entire batch.
|
||||
with self.lock, torch.inference_mode():
|
||||
for conversation in conversations:
|
||||
messages = prepare_messages(conversation, self.settings.max_pixels)
|
||||
text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
|
||||
images, _ = process_vision_info(messages, image_patch_size=16)
|
||||
inputs = self.processor(text=text, images=images, return_tensors="pt")
|
||||
count = inputs["input_ids"].shape[-1]
|
||||
if count > self.settings.max_tokens:
|
||||
bad(f"Input has {count} tokens; limit is {self.settings.max_tokens}. Input was not truncated.")
|
||||
last = self.processor.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0, -1].item())
|
||||
if last != "<embedding>":
|
||||
raise RuntimeError("Processor did not append the official <embedding> token.")
|
||||
inputs = inputs.to(self.device)
|
||||
# Reset explicitly as well, so older official code revisions are safe
|
||||
# when a text request follows a multimodal request.
|
||||
self.model.model.rope_deltas = None
|
||||
vector = self.model.embedding(**inputs, use_cache=False).float()
|
||||
vector = torch.nn.functional.normalize(vector[:, :dimension], dim=-1).cpu()[0]
|
||||
if vector.numel() != dimension or not torch.isfinite(vector).all() or vector.norm() < 0.99:
|
||||
raise RuntimeError("Invalid embedding output.")
|
||||
vectors.append(vector)
|
||||
total_tokens += count
|
||||
del inputs, images, messages
|
||||
return vectors, total_tokens
|
||||
|
||||
|
||||
class BodyLimit:
|
||||
"""Bound JSON bodies, including chunked requests, before parsing/base64 decoding."""
|
||||
def __init__(self, app, limit=32 * 1024 * 1024):
|
||||
self.app, self.limit = app, limit
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http" or scope["method"] != "POST":
|
||||
return await self.app(scope, receive, send)
|
||||
chunks, size = [], 0
|
||||
while True:
|
||||
message = await receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
return
|
||||
chunk = message.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > self.limit:
|
||||
response = JSONResponse({"error": {"message": "Request body exceeds 32 MiB."}}, status_code=413)
|
||||
return await response(scope, receive, send)
|
||||
chunks.append(chunk)
|
||||
if not message.get("more_body", False):
|
||||
break
|
||||
body = b"".join(chunks)
|
||||
|
||||
async def replay():
|
||||
nonlocal body
|
||||
if body is not None:
|
||||
value, body = body, None
|
||||
return {"type": "http.request", "body": value, "more_body": False}
|
||||
return await receive()
|
||||
|
||||
await self.app(scope, replay, send)
|
||||
|
||||
|
||||
def create_app(settings, engine_factory=Engine):
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
started = time.monotonic()
|
||||
app.state.engine = engine_factory(settings)
|
||||
log.info("Loaded %s on %s/%s in %.1fs", settings.name, app.state.engine.device,
|
||||
app.state.engine.dtype, time.monotonic() - started)
|
||||
yield
|
||||
del app.state.engine
|
||||
|
||||
app = FastAPI(title="WeMM Transformers embedding service", lifespan=lifespan)
|
||||
app.add_middleware(BodyLimit)
|
||||
|
||||
def authorize(authorization: str | None = Header(default=None)):
|
||||
expected = "Bearer " + settings.api_key
|
||||
if settings.api_key and not hmac.compare_digest((authorization or "").encode(), expected.encode()):
|
||||
raise HTTPException(401, "Invalid API key.", headers={"WWW-Authenticate": "Bearer"})
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def api_error(_request, exc):
|
||||
return JSONResponse(status_code=exc.status_code, headers=exc.headers,
|
||||
content={"error": {"message": exc.detail, "code": exc.status_code}})
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(_request, exc):
|
||||
return JSONResponse(status_code=400, content={"error": {
|
||||
"message": "Invalid request: " + exc.errors()[0]["msg"], "code": 400,
|
||||
}})
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
engine = app.state.engine
|
||||
return {"status": "ok", "model": settings.name, "backend": "transformers",
|
||||
"device": str(engine.device), "dtype": str(engine.dtype), "revision": settings.revision,
|
||||
"dimensions": engine.dimensions, "default_dimension": engine.default_dimension,
|
||||
"max_tokens": settings.max_tokens, "max_image_pixels": settings.max_pixels}
|
||||
|
||||
@app.get("/v1/models", dependencies=[Depends(authorize)])
|
||||
def models():
|
||||
return {"object": "list", "data": [{"id": settings.name, "object": "model", "created": 0, "owned_by": "tencent"}]}
|
||||
|
||||
@app.post("/v1/embeddings", dependencies=[Depends(authorize)])
|
||||
def embeddings(request: EmbedRequest):
|
||||
engine = app.state.engine
|
||||
conversations, dimension = parse_inputs(request, settings, engine.dimensions, engine.default_dimension)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
vectors, tokens = engine.encode(conversations, dimension)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("WeMM inference failed")
|
||||
raise HTTPException(500, "WeMM inference failed; inspect the server log.") from None
|
||||
data = []
|
||||
for index, vector in enumerate(vectors):
|
||||
value = (base64.b64encode(vector.numpy().astype("<f4").tobytes()).decode()
|
||||
if request.encoding_format == "base64" else vector.tolist())
|
||||
data.append({"object": "embedding", "index": index, "embedding": value})
|
||||
log.info("Embedded %d inputs, %d tokens, %d dimensions in %.2fs", len(data), tokens,
|
||||
dimension, time.monotonic() - started)
|
||||
return {"object": "list", "data": data, "model": settings.name,
|
||||
"usage": {"prompt_tokens": tokens, "total_tokens": tokens}}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model", default=Settings.model, help="WeMM Hugging Face ID or model directory")
|
||||
parser.add_argument("--served-model-name", help="API model ID; defaults to the repo ID or directory basename")
|
||||
parser.add_argument("--revision", help="Hugging Face revision (prefer a commit SHA for reproducibility)")
|
||||
parser.add_argument("--device", default="auto", help="auto, cuda, cuda:N, mps or cpu")
|
||||
parser.add_argument("--dtype", choices=("auto", "bfloat16", "float16", "float32"), default="auto")
|
||||
parser.add_argument("--max-tokens", type=int, default=8192)
|
||||
parser.add_argument("--max-pixels", type=int, default=1048576)
|
||||
parser.add_argument("--max-batch-size", type=int, default=32)
|
||||
parser.add_argument("--cpu-threads", type=int, default=8)
|
||||
parser.add_argument("--local-files-only", action="store_true")
|
||||
parser.add_argument("--mps-fallback", action="store_true", help="Allow PyTorch CPU fallback for unsupported MPS ops")
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=18080)
|
||||
args = vars(parser.parse_args())
|
||||
host, port = args.pop("host"), args.pop("port")
|
||||
if args.pop("mps_fallback"):
|
||||
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
|
||||
settings = Settings(**args, api_key=os.environ.get("WEMM_API_KEY", ""))
|
||||
import uvicorn
|
||||
# A single process owns the model; do not add workers without separate GPUs.
|
||||
uvicorn.run(create_app(settings), host=host, port=port, workers=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Check a running WeMM service with actual text and image inference."""
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
def dot(a, b):
|
||||
return sum(x * y for x, y in zip(a, b))
|
||||
|
||||
|
||||
def text_message(text):
|
||||
return [{"role": "user", "content": [{"type": "text", "text": text}]}]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:18080", help="Service root URL, without /v1")
|
||||
parser.add_argument("--timeout", type=float, default=180)
|
||||
args = parser.parse_args()
|
||||
headers = {"Authorization": "Bearer " + os.environ["WEMM_API_KEY"]} if os.environ.get("WEMM_API_KEY") else {}
|
||||
results = []
|
||||
with httpx.Client(base_url=args.base_url.rstrip("/"), timeout=args.timeout, headers=headers) as client:
|
||||
health_response = client.get("/health")
|
||||
health_response.raise_for_status()
|
||||
health = health_response.json()
|
||||
model = health["model"]
|
||||
default_dimension = health["default_dimension"]
|
||||
response = client.get("/v1/models")
|
||||
response.raise_for_status()
|
||||
assert model in [row["id"] for row in response.json()["data"]]
|
||||
|
||||
def run(name, **payload):
|
||||
started = time.monotonic()
|
||||
response = client.post("/v1/embeddings", json={"model": model, **payload})
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
vectors = [row["embedding"] for row in data["data"]]
|
||||
assert [row["index"] for row in data["data"]] == list(range(len(vectors)))
|
||||
for vector in vectors:
|
||||
assert len(vector) == payload.get("dimensions", default_dimension)
|
||||
assert all(math.isfinite(x) for x in vector)
|
||||
assert abs(math.sqrt(dot(vector, vector)) - 1) < 1e-4
|
||||
results.append({"test": name, "seconds": round(time.monotonic() - started, 3),
|
||||
"vectors": len(vectors), "dimension": len(vectors[0])})
|
||||
print(json.dumps(results[-1]), flush=True)
|
||||
return vectors
|
||||
|
||||
texts = ["A red square on a white background.", "A blue circle on a black background."]
|
||||
text_vectors = run("batched text messages", messages=[text_message(text) for text in texts])
|
||||
images = []
|
||||
for color, shape, background in [("red", "rectangle", "white"), ("blue", "ellipse", "black")]:
|
||||
image = Image.new("RGB", (256, 256), background)
|
||||
getattr(ImageDraw.Draw(image), shape)((40, 40, 216, 216), fill=color)
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
images.append([{"role": "user", "content": [{"type": "image_url", "image_url": {
|
||||
"url": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode(),
|
||||
}}]}])
|
||||
image_vectors = run("batched image messages", messages=images)
|
||||
scores = [[dot(text, image) for image in image_vectors] for text in text_vectors]
|
||||
assert scores[0][0] > scores[0][1] and scores[1][1] > scores[1][0], scores
|
||||
repeated = run("text after image state isolation", messages=text_message(texts[0]))[0]
|
||||
assert dot(repeated, text_vectors[0]) > 0.9999
|
||||
dimension = min(health["dimensions"])
|
||||
short = run("Matryoshka dimensions", messages=text_message(texts[0]), dimensions=dimension)[0]
|
||||
prefix = text_vectors[0][:dimension]
|
||||
norm = math.sqrt(dot(prefix, prefix))
|
||||
assert dot(short, [value / norm for value in prefix]) > 0.9999
|
||||
raw = run("raw text compatibility", input=texts[0])[0]
|
||||
assert dot(raw, text_vectors[0]) > 0.9999
|
||||
for payload in ({"input": "hello", "dimensions": 3}, {"input": "a", "messages": []},
|
||||
{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
}}]}]}):
|
||||
response = client.post("/v1/embeddings", json={"model": model, **payload})
|
||||
assert response.status_code == 400, response.text
|
||||
print(json.dumps({"status": "passed", "health": health, "tests": results, "text_image_scores": scores}), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
"""Contract tests without downloading weights: python -m unittest discover -s scripts/wemm-transformers."""
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import struct
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
from serve import BodyLimit, Message, Settings, create_app, decode_image, prepare_messages, resolve_device_dtype
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
device, dtype = "cpu", "float32"
|
||||
|
||||
def __init__(self, settings, dimensions=(64, 128, 256, 512, 1024, 2048)):
|
||||
self.dimensions = dimensions
|
||||
self.default_dimension = dimensions[-1]
|
||||
self.calls = []
|
||||
|
||||
def encode(self, conversations, dimension):
|
||||
self.calls.append(conversations)
|
||||
vectors = []
|
||||
for index, _ in enumerate(conversations):
|
||||
vector = torch.zeros(dimension)
|
||||
vector[index] = 1
|
||||
vectors.append(vector)
|
||||
return vectors, len(conversations) * 10
|
||||
|
||||
|
||||
class ServiceTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.settings = Settings()
|
||||
self.engine = FakeEngine(self.settings)
|
||||
self.client = self.enterContext(TestClient(create_app(self.settings, lambda _: self.engine)))
|
||||
|
||||
def post(self, **payload):
|
||||
return self.client.post("/v1/embeddings", json={"model": self.settings.name, **payload})
|
||||
|
||||
def test_input_and_single_or_batched_messages(self):
|
||||
message = {"role": "user", "content": [{"type": "text", "text": "hello"}]}
|
||||
for payload, count in [({"input": "hello"}, 1), ({"input": ["a", "b"]}, 2),
|
||||
({"messages": [message]}, 1), ({"messages": [[message], [message]]}, 2)]:
|
||||
with self.subTest(payload=payload):
|
||||
response = self.post(**payload)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
data = response.json()["data"]
|
||||
self.assertEqual([x["index"] for x in data], list(range(count)))
|
||||
self.assertTrue(all(len(x["embedding"]) == 2048 for x in data))
|
||||
self.assertEqual([x["embedding"][i] for i, x in enumerate(data)], [1] * count)
|
||||
|
||||
def test_all_model_sizes_use_their_own_default_and_allowed_dimensions(self):
|
||||
for size, dimensions in [("2B", (64, 128, 256, 512, 1024, 2048)),
|
||||
("4B", (64, 128, 256, 512, 1024, 2560)),
|
||||
("9B", (64, 128, 256, 512, 1024, 2048, 4096))]:
|
||||
settings = Settings(model="tencent/WeMM-Embedding-" + size)
|
||||
with self.subTest(size=size), TestClient(create_app(settings, lambda s: FakeEngine(s, dimensions))) as client:
|
||||
for requested in (None, 1024):
|
||||
payload = {"model": settings.name, "input": "hello"}
|
||||
if requested is not None:
|
||||
payload["dimensions"] = requested
|
||||
response = client.post("/v1/embeddings", json=payload)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
self.assertEqual(len(response.json()["data"][0]["embedding"]), requested or dimensions[-1])
|
||||
payload["dimensions"] = 2049
|
||||
self.assertEqual(client.post("/v1/embeddings", json=payload).status_code, 400)
|
||||
self.assertEqual(client.get("/health").json()["default_dimension"], dimensions[-1])
|
||||
|
||||
def test_base64_is_little_endian_float32(self):
|
||||
response = self.post(input="hello", dimensions=64, encoding_format="base64")
|
||||
vector = struct.unpack("<64f", base64.b64decode(response.json()["data"][0]["embedding"]))
|
||||
self.assertEqual(vector, (1,) + (0,) * 63)
|
||||
|
||||
def test_invalid_requests_never_reach_inference(self):
|
||||
for payload in [dict(input="a", messages=[]), dict(input=[]), dict(messages=[[]]),
|
||||
dict(input="a", dimensions=True), dict(input="a", dimensions=0),
|
||||
dict(input="a", temperature=0), dict(input=["a"] * 33),
|
||||
dict(messages=[{"role": "user", "content": [{"type": "video", "video": "x"}]}]),
|
||||
dict(messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]}])]:
|
||||
with self.subTest(payload=payload):
|
||||
self.assertEqual(self.post(**payload).status_code, 400)
|
||||
self.assertEqual(self.post(model="wrong", input="a").status_code, 404)
|
||||
self.assertEqual(self.engine.calls, [])
|
||||
|
||||
def test_authentication_on_models_and_embeddings(self):
|
||||
settings = Settings(api_key="test-only-key")
|
||||
with TestClient(create_app(settings, lambda _: self.engine)) as client:
|
||||
self.assertEqual(client.get("/health").status_code, 200)
|
||||
self.assertEqual(client.get("/v1/models").status_code, 401)
|
||||
for token, status in [("Bearer wrong", 401), ("Bearer test-only-key", 200)]:
|
||||
response = client.post("/v1/embeddings", json={"model": settings.name, "input": "a"},
|
||||
headers={"Authorization": token})
|
||||
self.assertEqual(response.status_code, status)
|
||||
self.assertEqual(len(self.engine.calls), 1)
|
||||
|
||||
def test_image_decoding_and_invalid_bytes(self):
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (64, 64), "red").save(buffer, format="PNG")
|
||||
uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
self.assertEqual(decode_image(uri).size, (64, 64))
|
||||
for data in ["!invalid!", base64.b64encode(b"not an image").decode()]:
|
||||
with self.assertRaises(HTTPException) as error:
|
||||
decode_image("data:image/png;base64," + data)
|
||||
self.assertEqual(error.exception.status_code, 400)
|
||||
|
||||
def test_transparent_images_match_official_white_background(self):
|
||||
from qwen_vl_utils import process_vision_info
|
||||
image = Image.new("RGBA", (256, 256), (0, 0, 0, 0))
|
||||
image.putpixel((128, 128), (255, 0, 0, 128))
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
conversation = [Message.model_validate({"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": uri}},
|
||||
]})]
|
||||
actual, _ = process_vision_info(prepare_messages(conversation, 1048576), image_patch_size=16)
|
||||
expected, _ = process_vision_info([{"role": "user", "content": [
|
||||
{"type": "image", "image": image, "max_pixels": 1048576},
|
||||
]}], image_patch_size=16)
|
||||
self.assertEqual(actual[0].getpixel((0, 0)), (255, 255, 255))
|
||||
self.assertEqual(actual[0].getpixel((128, 128)), (255, 127, 127))
|
||||
self.assertEqual(actual[0].tobytes(), expected[0].tobytes())
|
||||
|
||||
def test_cpu_defaults_and_explicit_unavailable_device(self):
|
||||
with patch.object(torch.cuda, "is_available", return_value=False), patch.object(torch.backends.mps, "is_available", return_value=False):
|
||||
device, dtype = resolve_device_dtype(torch, Settings())
|
||||
self.assertEqual(str(device), "cpu")
|
||||
self.assertEqual(dtype, torch.float32)
|
||||
for requested in ("cuda", "mps"):
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_device_dtype(torch, Settings(device=requested))
|
||||
|
||||
def test_cuda_auto_dtype_and_explicit_bf16_guard(self):
|
||||
with patch.object(torch.cuda, "is_available", return_value=True), patch.object(torch.cuda, "device", return_value=nullcontext()):
|
||||
for supported, expected in [(True, torch.bfloat16), (False, torch.float16)]:
|
||||
with patch.object(torch.cuda, "is_bf16_supported", return_value=supported):
|
||||
device, dtype = resolve_device_dtype(torch, Settings())
|
||||
self.assertEqual(str(device), "cuda")
|
||||
self.assertEqual(dtype, expected)
|
||||
if not supported:
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_device_dtype(torch, Settings(device="cuda", dtype="bfloat16"))
|
||||
|
||||
def test_body_limit_counts_chunked_bodies(self):
|
||||
async def run(chunks):
|
||||
received, sent = [], []
|
||||
events = iter({"type": "http.request", "body": chunk, "more_body": i < len(chunks) - 1}
|
||||
for i, chunk in enumerate(chunks))
|
||||
async def receive():
|
||||
return next(events)
|
||||
async def send(event):
|
||||
sent.append(event)
|
||||
async def downstream(scope, receive, send):
|
||||
received.append((await receive())["body"])
|
||||
await BodyLimit(downstream, limit=4)({"type": "http", "method": "POST"}, receive, send)
|
||||
return received, sent
|
||||
received, _ = asyncio.run(run([b"12", b"34"]))
|
||||
self.assertEqual(received, [b"1234"])
|
||||
received, sent = asyncio.run(run([b"12", b"345"]))
|
||||
self.assertEqual(received, [])
|
||||
self.assertEqual(sent[0]["status"], 413)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -95,6 +95,46 @@
|
|||
| 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` |
|
||||
| 自建重排服务返回的是未归一化分数(logit) | `{"score_scale": "logit"}` |
|
||||
|
||||
#### WeMM-Embedding
|
||||
|
||||
自定义厂商(`generic`)的向量模型列表已内置以下预设,选择后会自动识别图片输入和原生向量维度:
|
||||
|
||||
| 模型名称 | 原生维度 |
|
||||
| --- | --- |
|
||||
| `tencent/WeMM-Embedding-2B` | 2048 |
|
||||
| `tencent/WeMM-Embedding-4B` | 2560 |
|
||||
| `tencent/WeMM-Embedding-9B` | 4096 |
|
||||
|
||||
可选择官方 vLLM 部署,或使用本仓库的 [Transformers 部署脚本](../06-development/06-wemm-transformers.md)在 CUDA、MPS 或 CPU 上提供同一 Embedding 接口。MPS 是可选计算设备,服务也可以部署在独立服务器上。
|
||||
|
||||
按[官方部署说明](https://github.com/Tencent/WeMM-Embedding#serving)使用 vLLM `0.27.0`,并加载模型目录中的 `embedding_chat_template.jinja`。例如部署 2B,并通过 `--served-model-name` 让服务名称与预设一致:
|
||||
|
||||
```bash
|
||||
MODEL_PATH=/path/to/WeMM-Embedding-2B
|
||||
vllm serve "$MODEL_PATH" \
|
||||
--runner pooling \
|
||||
--chat-template "$MODEL_PATH/embedding_chat_template.jinja" \
|
||||
--served-model-name tencent/WeMM-Embedding-2B
|
||||
```
|
||||
|
||||
4B、9B 分别替换模型路径与服务名称。在 WeKnora 中选择「自定义」厂商,Base URL 填写服务的 `http://主机:端口/v1`,选择对应向量模型,初次接入保持「自定义输出维度」关闭。若服务没有设置 `--served-model-name`,模型名称须填写服务实际返回的本地路径;短名称 `WeMM-Embedding-2B` 及以 `/WeMM-Embedding-2B` 结尾的路径也能识别预设,4B、9B 同理。任意自定义别名需要另行配置模型能力和协议覆盖。
|
||||
|
||||
预设让文本和图片都通过 `messages` 进入模型模板,由模板追加取向量所需的 `<embedding>` 标记;启用 `batch_messages`,将同一批内的独立 conversations 合并请求,每条仍返回各自的向量。图片批量调用最多每批 5 张,超出自动拆分;这是 WeKnora 选择的批量大小,与现有 Qwen3-VL 预设一致,不代表模型或 vLLM 的官方上限,可用 `max_image_batch_size` 按服务承载能力调整。当前知识库图片索引任务仍逐张调用,不会因该配置自动合并多个任务。不发送历史默认的 `truncate_prompt_tokens: 511`;超出服务上下文上限时由服务报错,不会静默截断。无需额外填写兼容 JSON 或图片提示词。若此前用普通文本 `input` 为该模型建立过索引,需要重建向量。
|
||||
|
||||
WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_tokens: false`、`add_generation_prompt: false` 默认行为,不要照搬 Qwen3-VL 预设的 `add_special_tokens: true`。vLLM 的 Embedding 默认使用末尾 token 并做 L2 归一化,无需额外发送 `normalize` 或关闭 `use_activation`。
|
||||
|
||||
需要降维时,开启「自定义输出维度」并选择模型配置中的合法维度,WeKnora 会给文本和图片请求发送相同的 `dimensions`;vLLM 会先截取对应维度,再归一化:
|
||||
|
||||
| 模型 | 可选维度 |
|
||||
| --- | --- |
|
||||
| 2B | 64、128、256、512、1024、2048 |
|
||||
| 4B | 64、128、256、512、1024、2560 |
|
||||
| 9B | 64、128、256、512、1024、2048、4096 |
|
||||
|
||||
这些值来自[官方模型配置与说明](https://github.com/Tencent/WeMM-Embedding#model-zoo),vLLM 会拒绝列表之外的维度。更换输出维度后需重建已有向量索引。超时、并发与图片大小限制保持现有默认值,按实际服务性能调整。
|
||||
|
||||
知识库还需开启图片向量索引才能直接检索图片内容。当前预设接入文本和图片;视频、任意图文交错输入及扫描 PDF 整页向量化不在当前业务链路的支持范围内。
|
||||
|
||||
#### 字段参考
|
||||
|
||||
下表的「协议默认」是没有任何厂商、目录或覆盖时的取值;具体厂商的实际值以「实际调用方式」面板为准。
|
||||
|
|
|
|||
|
|
@ -0,0 +1,205 @@
|
|||
# WeMM Transformers 部署
|
||||
|
||||
使用 [WeMM 官方 Transformers 推理代码](https://github.com/Tencent/WeMM-Embedding#transformers)加载模型,由仓库内的 `scripts/wemm-transformers/serve.py` 提供 HTTP Embedding 服务。服务可以与 WeKnora 部署在同一机器,也可以部署在独立 GPU 服务器上,通过 Base URL 接入。
|
||||
|
||||
CUDA、Apple MPS 和 CPU 是可选的计算设备,使用同一份脚本和请求协议。此 HTTP 适配层由 WeKnora 提供,支持文本和图片;模型本身支持的视频等能力尚未在此接口开放。官方 vLLM 部署方式见[模型配置](../03-features/06-models.md#wemm-embedding)。
|
||||
|
||||
## 安装依赖
|
||||
|
||||
以下命令在 WeKnora 仓库根目录执行,使用 Python 3.12。虚拟环境和模型文件放在仓库外:
|
||||
|
||||
```bash
|
||||
python3.12 -m venv "$HOME/.venvs/weknora-wemm"
|
||||
source "$HOME/.venvs/weknora-wemm/bin/activate"
|
||||
python -m pip install --upgrade pip
|
||||
```
|
||||
|
||||
按设备选择一组 PyTorch 安装命令。以下使用已验证的 PyTorch 2.10.0 / torchvision 0.25.0,CUDA wheel 须与驱动兼容;其他 CUDA 版本可参考 [PyTorch 安装说明](https://pytorch.org/get-started/previous-versions/#v2100)。
|
||||
|
||||
```bash
|
||||
# NVIDIA GPU:示例为 CUDA 12.8 wheel
|
||||
python -m pip install torch==2.10.0 torchvision==0.25.0 \
|
||||
--index-url https://download.pytorch.org/whl/cu128
|
||||
|
||||
# 或 Apple Silicon / MPS
|
||||
python -m pip install torch==2.10.0 torchvision==0.25.0
|
||||
|
||||
# 或 Linux / Windows CPU wheel
|
||||
python -m pip install torch==2.10.0 torchvision==0.25.0 \
|
||||
--index-url https://download.pytorch.org/whl/cpu
|
||||
```
|
||||
|
||||
安装其余依赖:
|
||||
|
||||
```bash
|
||||
python -m pip install -r scripts/wemm-transformers/requirements.txt
|
||||
```
|
||||
|
||||
Transformers 固定为官方推荐的 `5.2.0`,`qwen-vl-utils` 固定为 `0.0.14`。不要随意替换 Transformers 版本,避免预处理行为变化。脚本会启用 `trust_remote_code=True` 来加载 WeMM 的 `embedding()` 实现,应使用官方权重或已审查的模型目录。
|
||||
|
||||
## 启动服务
|
||||
|
||||
以 CUDA 上的 2B 为例:
|
||||
|
||||
```bash
|
||||
python scripts/wemm-transformers/serve.py \
|
||||
--model tencent/WeMM-Embedding-2B \
|
||||
--revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \
|
||||
--device cuda \
|
||||
--host 127.0.0.1 \
|
||||
--port 18080
|
||||
```
|
||||
|
||||
首次启动会从 Hugging Face 下载模型到标准缓存,后续复用缓存。`--revision` 可固定模型代码与权重版本;示例 SHA 属于 2B,切换 4B、9B 时应选取对应仓库的版本,不能沿用此 SHA。未指定时使用该仓库默认版本。
|
||||
|
||||
已有模型目录时可以离线启动,并指定 WeKnora 调用时使用的模型名称:
|
||||
|
||||
```bash
|
||||
python scripts/wemm-transformers/serve.py \
|
||||
--model /srv/models/WeMM-Embedding-2B \
|
||||
--served-model-name tencent/WeMM-Embedding-2B \
|
||||
--local-files-only \
|
||||
--device cuda
|
||||
```
|
||||
|
||||
也可以提前下载并固定版本,例如:
|
||||
|
||||
```bash
|
||||
hf download tencent/WeMM-Embedding-2B \
|
||||
--revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \
|
||||
--local-dir /srv/models/WeMM-Embedding-2B
|
||||
```
|
||||
|
||||
其他设备只需更换参数:
|
||||
|
||||
```bash
|
||||
# Apple Silicon,允许不支持的 MPS 算子回退到 CPU
|
||||
python scripts/wemm-transformers/serve.py \
|
||||
--model tencent/WeMM-Embedding-2B --device mps --mps-fallback
|
||||
|
||||
# CPU,默认使用 float32
|
||||
python scripts/wemm-transformers/serve.py \
|
||||
--model tencent/WeMM-Embedding-2B --device cpu
|
||||
```
|
||||
|
||||
4B、9B 将 `--model` 换成对应 ID 或路径。服务从模型配置读取原生维度及 Matryoshka 可选维度,不会将其他规格强制按 2048 维处理:
|
||||
|
||||
| 模型 ID | 原生维度 |
|
||||
| --- | --- |
|
||||
| `tencent/WeMM-Embedding-2B` | 2048 |
|
||||
| `tencent/WeMM-Embedding-4B` | 2560 |
|
||||
| `tencent/WeMM-Embedding-9B` | 4096 |
|
||||
|
||||
服务以前台方式运行,按 Ctrl+C 停止。常驻部署时由 systemd、容器或其他进程管理器托管同一命令。一个进程加载一份完整模型到一个设备,不提供多 GPU 切分;选择规格前应预留权重、视觉处理和推理激活所需的显存/内存。
|
||||
|
||||
### 独立服务器与鉴权
|
||||
|
||||
跨机器部署时使用 `--host 0.0.0.0`,并在服务进程环境中设置 `WEMM_API_KEY`。`/v1/models` 与 `/v1/embeddings` 会验证 `Authorization: Bearer <key>`;WeKnora 的 API Key 填写相同值。服务不直接终止 TLS,跨不可信网络时通过 HTTPS 反向代理访问。`/health` 仅提供运行状态和非敏感模型元数据,不要求 API Key。
|
||||
|
||||
默认监听回环地址、未设置 API Key。WeKnora 若运行在 Docker 中,`127.0.0.1` 指向 WeKnora 容器自身,应填写容器能够访问的推理服务地址,例如同一容器网络的服务名,或已配置的宿主机地址。私网地址还需符合 WeKnora 的 SSRF 访问策略,配置方法见[安全说明](../01-getting-started/04-configuration.md)。
|
||||
|
||||
## WeKnora 接入
|
||||
|
||||
在「设置 → 模型管理」添加:
|
||||
|
||||
| 字段 | 填写 |
|
||||
| --- | --- |
|
||||
| 类型 | Embedding |
|
||||
| 来源 / 服务商 | API / 自定义(OpenAI 兼容接口) |
|
||||
| 模型名称 | `/v1/models` 返回的 ID,推荐保留 `tencent/WeMM-Embedding-2B` 等官方名称 |
|
||||
| Base URL | `http://推理服务地址:18080/v1`,使用 HTTPS 代理时填写对应地址 |
|
||||
| API Key | 设置了 `WEMM_API_KEY` 时填写相同值 |
|
||||
| 图片输入 | 官方名称由预设识别;自定义别名需手动开启 |
|
||||
| 自定义输出维度 | 初次接入关闭,让连接检测识别原生维度;需要降维时再开启 |
|
||||
| 后台并发上限 | 建议从 1 开始 |
|
||||
|
||||
现有 WeMM 预设的 `messages` 格式同时适用于此服务。推荐在「高级 → 协议兼容覆盖」设置较小批量和较长超时:
|
||||
|
||||
```json
|
||||
{
|
||||
"api": "openai-embeddings",
|
||||
"image_format": "messages",
|
||||
"text_as_messages": true,
|
||||
"batch_messages": true,
|
||||
"accepts_truncate_prompt_tokens": false,
|
||||
"max_batch_size": 4,
|
||||
"max_image_batch_size": 2,
|
||||
"request_timeout_seconds": 180
|
||||
}
|
||||
```
|
||||
|
||||
完整配置也适用于尚未包含 WeMM 预设的旧版本或自定义模型别名;此时应手动打开图片输入。点击「测试连接」,确认实际维度后保存。知识库还需开启图片向量索引,配置方式见[模型配置](../03-features/06-models.md)。切换模型或输出维度需要重建已有索引。
|
||||
|
||||
## 请求格式与推理流程
|
||||
|
||||
接口为 `POST /v1/embeddings`。纯文本可以使用 `input`:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "tencent/WeMM-Embedding-2B",
|
||||
"input": ["第一段文本", "第二段文本"],
|
||||
"encoding_format": "float"
|
||||
}
|
||||
```
|
||||
|
||||
文本和图片也可使用 `messages`。以下二维数组包含两个独立输入,分别返回一个文本向量和一个图片向量:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "tencent/WeMM-Embedding-2B",
|
||||
"dimensions": 1024,
|
||||
"messages": [
|
||||
[{"role": "user", "content": [{"type": "text", "text": "白色背景上的红色方块"}]}],
|
||||
[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,图片Base64"}}]}]
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`input` 和 `messages` 只能选一个。单条 conversation 可以使用一维消息数组;文字与图片放在同一 conversation 中会得到一个联合向量。返回值采用 OpenAI Embedding 的 `data[].embedding`、`data[].index` 和 `usage` 结构,编码支持 `float` 与 float32 小端字节的 `base64`。
|
||||
|
||||
服务会将文本包装成消息,把内联图片解码成 PIL 图片,使用官方 processor 的聊天模板,再转换为张量。Tokenizer 自动追加 `<embedding>`,调用方无需手动添加。推理调用 `model.embedding(..., use_cache=False)`,随后按请求维度截取并做 L2 归一化。这里使用 Transformers 自带的模型 tokenizer/processor;vLLM 的 `embedding_chat_template.jinja` 不作为本脚本的额外模板传入。
|
||||
|
||||
HTTP 支持批量输入,但模型逐条执行,并串行处理推理请求,以隔离模型内部的可变位置状态。`batch_messages: true` 表示合并 HTTP 请求,不代表 GPU 张量批处理,也不提供 vLLM 的连续批处理调度。
|
||||
|
||||
## 参数与限制
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `--device` | `auto` | 自动按 CUDA → MPS → CPU 选择;可显式指定 `cuda:N`、`mps`、`cpu`,指定不可用设备会报错 |
|
||||
| `--dtype` | `auto` | CUDA 支持 BF16 时用 BF16,否则 FP16;MPS 用 BF16;CPU 用 FP32。可显式指定 `bfloat16`、`float16`、`float32` |
|
||||
| `--max-tokens` | 8192 | 每个输入经过模板和图片处理后的总 token 上限,超限拒绝,不静默截断 |
|
||||
| `--max-pixels` | 1048576 | 每张图片缩放后的像素上限,至少 65536;较小上限会影响高分辨率文档效果 |
|
||||
| `--max-batch-size` | 32 | 每个 HTTP 请求的独立输入数上限;WeKnora 的批量设置不应超过它 |
|
||||
| `--cpu-threads` | 8 | PyTorch CPU 线程数 |
|
||||
| `--mps-fallback` | 关闭 | 允许 PyTorch 将不支持的 MPS 算子放到 CPU 执行 |
|
||||
| `--local-files-only` | 关闭 | 仅使用本机文件或已有缓存,不下载权重 |
|
||||
| `--host` / `--port` | `127.0.0.1` / `18080` | 监听地址与端口 |
|
||||
|
||||
每个请求体最多 32 MiB,包含分块传输;每张图片的 data URI 最多 16 MiB,解码前原图最多 4000 万像素,每个 conversation 最多 5 张图片。图片只接受内联 Base64,不抓取远程 URL 或文件路径。`image_url.detail` 仅为格式兼容,实际分辨率由 `--max-pixels` 控制。所有上限都是此部署脚本的资源设置,不代表 WeMM 模型本身的上限。
|
||||
|
||||
## 验证与排障
|
||||
|
||||
运行无需下载权重的接口测试:
|
||||
|
||||
```bash
|
||||
python -m unittest discover -s scripts/wemm-transformers -v
|
||||
```
|
||||
|
||||
对运行中的服务做实际图文推理测试:
|
||||
|
||||
```bash
|
||||
python scripts/wemm-transformers/smoke_test.py \
|
||||
--base-url http://127.0.0.1:18080
|
||||
```
|
||||
|
||||
设置了鉴权时,测试进程同样读取 `WEMM_API_KEY`。测试覆盖批量图文、输出维度与归一化、图文匹配、图片后文本状态隔离及降维;输出耗时和结果到标准输出,不修改知识库。
|
||||
|
||||
当前在 Apple M4 Pro 上完成了 **WeMM 2B / MPS / BF16** 和 **WeMM 2B / CPU / FP32** 的完整图文推理测试。2B、4B、9B 的接口维度处理及 CUDA 精度选择有自动化覆盖;4B/9B 的完整权重推理和 CUDA 部署仍需在目标设备运行上述验证脚本。不要将接口测试视为这些设备和规格的性能验证。
|
||||
|
||||
常见问题:
|
||||
|
||||
- **CUDA/MPS 不可用**:检查 PyTorch wheel、驱动与设备,或显式改为 `--device cpu`;脚本不会将显式设备要求静默降级。
|
||||
- **MPS 算子不支持**:尝试 `--mps-fallback`;BF16 不可用时可使用 `--dtype float32`,同时检查内存余量。
|
||||
- **显存不足或超时**:减小模型规格、图片像素、输入长度和 WeKnora 批量。该服务为单 worker,增加 HTTP 并发不会自动增加推理吞吐。
|
||||
- **模型加载缓慢**:首次需要下载权重;先预下载并固定版本,离线启动时检查目录中同时包含权重、tokenizer、processor 和官方 Python 模型代码。
|
||||
- **模型名不匹配**:检查 `/v1/models`;使用自定义目录时建议显式设置 `--served-model-name`。
|
||||
Loading…
Reference in New Issue