feat(models): add WeMM embedding presets and Transformers serving (#4238)

main
lyingbug 2026-10-13 17:22:24 +08:00 committed by GitHub
parent 8af7938aed
commit 94fae6de43
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 1169 additions and 0 deletions

View File

@ -2185,6 +2185,75 @@
"add_special_tokens": true
}
}
},
{
"id": "tencent/WeMM-Embedding-2B",
"match": "*/WeMM-Embedding-2B",
"type": "Embedding",
"name": "WeMM-Embedding-2B",
"aliases": [
"WeMM-Embedding-2B"
],
"input": [
"text",
"image"
],
"dimension": 2048,
"source": "https://huggingface.co/tencent/WeMM-Embedding-2B",
"compat": {
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_image_batch_size": 5
}
},
{
"id": "tencent/WeMM-Embedding-4B",
"match": "*/WeMM-Embedding-4B",
"type": "Embedding",
"name": "WeMM-Embedding-4B",
"aliases": [
"WeMM-Embedding-4B"
],
"input": [
"text",
"image"
],
"dimension": 2560,
"source": "https://huggingface.co/tencent/WeMM-Embedding-4B",
"compat": {
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_image_batch_size": 5
}
},
{
"id": "tencent/WeMM-Embedding-9B",
"match": "*/WeMM-Embedding-9B",
"type": "Embedding",
"name": "WeMM-Embedding-9B",
"aliases": [
"WeMM-Embedding-9B"
],
"input": [
"text",
"image"
],
"dimension": 4096,
"source": "https://huggingface.co/tencent/WeMM-Embedding-9B",
"compat": {
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_image_batch_size": 5
}
}
],
"gpustack": [],

View File

@ -1050,6 +1050,45 @@
"add_special_tokens": true
}
}
},
{
"id": "tencent/WeMM-Embedding-2B",
"match": "*/WeMM-Embedding-2B",
"type": "Embedding",
"compat": {
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_image_batch_size": 5
}
},
{
"id": "tencent/WeMM-Embedding-4B",
"match": "*/WeMM-Embedding-4B",
"type": "Embedding",
"compat": {
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_image_batch_size": 5
}
},
{
"id": "tencent/WeMM-Embedding-9B",
"match": "*/WeMM-Embedding-9B",
"type": "Embedding",
"compat": {
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_image_batch_size": 5
}
}
],
"hunyuan": [

View File

@ -1454,6 +1454,51 @@
"image"
],
"source": "https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B"
},
{
"id": "tencent/WeMM-Embedding-2B",
"match": "*/WeMM-Embedding-2B",
"type": "Embedding",
"name": "WeMM-Embedding-2B",
"aliases": [
"WeMM-Embedding-2B"
],
"input": [
"text",
"image"
],
"dimension": 2048,
"source": "https://huggingface.co/tencent/WeMM-Embedding-2B"
},
{
"id": "tencent/WeMM-Embedding-4B",
"match": "*/WeMM-Embedding-4B",
"type": "Embedding",
"name": "WeMM-Embedding-4B",
"aliases": [
"WeMM-Embedding-4B"
],
"input": [
"text",
"image"
],
"dimension": 2560,
"source": "https://huggingface.co/tencent/WeMM-Embedding-4B"
},
{
"id": "tencent/WeMM-Embedding-9B",
"match": "*/WeMM-Embedding-9B",
"type": "Embedding",
"name": "WeMM-Embedding-9B",
"aliases": [
"WeMM-Embedding-9B"
],
"input": [
"text",
"image"
],
"dimension": 4096,
"source": "https://huggingface.co/tencent/WeMM-Embedding-9B"
}
],
"gpustack": [],

View File

@ -0,0 +1,127 @@
package embedding
import (
"context"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
modelruntime "github.com/Tencent/WeKnora/internal/models/runtime"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
// WeMM's vLLM template appends the dedicated <embedding> pooling token.
// Both retrieval sides and images must use that template, including when
// vLLM exposes the local model path instead of the Hugging Face repository ID.
// https://huggingface.co/tencent/WeMM-Embedding-2B/blob/main/embedding_chat_template.jinja
func TestWeMMVLLMPresets(t *testing.T) {
vendor, ok := modelruntime.Get("generic")
require.True(t, ok)
listed := vendor.ModelsByType(types.ModelTypeEmbedding)
for _, tc := range []struct {
size string
dims int
}{{"2B", 2048}, {"4B", 2560}, {"9B", 4096}} {
short := "WeMM-Embedding-" + tc.size
id := "tencent/" + short
t.Run(tc.size, func(t *testing.T) {
visible := false
for _, entry := range listed {
if entry.ID == id {
visible = true
require.Equal(t, tc.dims, entry.Dimension)
}
}
require.True(t, visible, "preset must appear in the embedding picker")
for _, name := range []string{id, short, "/models/" + short, strings.ToLower(id)} {
t.Run(name, func(t *testing.T) {
resolved, err := modelruntime.Resolve(modelruntime.Ref{
Provider: "generic", Model: name, ModelType: types.ModelTypeEmbedding,
})
require.NoError(t, err)
require.True(t, resolved.Cataloged)
require.Equal(t, tc.dims, resolved.Spec.Dimension)
require.Equal(t, api.EmbeddingOpenAI, resolved.EmbeddingAPI)
require.Equal(t, []string{"text", "image"}, resolved.Spec.Input)
require.True(t, resolved.Embeddings.BatchMessages)
require.Equal(t, 5, resolved.Embeddings.ImageBatchLimit())
up := newUpstream(t)
e, err := NewEmbedder(Config{
Source: types.ModelSourceRemote, Provider: "generic", ModelName: name,
BaseURL: up.url + "/v1", Dimensions: tc.dims, TruncatePromptTokens: 511,
}, nil, nil)
require.NoError(t, err)
require.Equal(t, tc.dims, e.GetDimensions())
ie, ok := AsImageEmbedder(e)
require.True(t, ok, "images must work without a row override")
contexts := []context.Context{context.Background(), types.WithEmbedQuery(context.Background())}
for _, ctx := range contexts {
vectors, err := e.BatchEmbed(ctx, []string{"ab", "c"})
require.NoError(t, err)
require.Equal(t, [][]float32{{2}, {1}}, vectors)
}
vectors, err := ie.BatchEmbedImages(context.Background(), pngs(6))
require.NoError(t, err)
require.Equal(t, [][]float32{{1}, {2}, {3}, {4}, {5}, {6}}, vectors)
conversation := func(part map[string]any) []any {
return []any{map[string]any{"role": "user", "content": []any{part}}}
}
textBatch := []any{
conversation(map[string]any{"type": "text", "text": "ab"}),
conversation(map[string]any{"type": "text", "text": "c"}),
}
imageBatch := []any{}
for _, img := range pngs(5) {
imageBatch = append(imageBatch, conversation(map[string]any{
"type": "image_url", "image_url": map[string]any{"url": img.DataURI()},
}))
}
messages := []any{textBatch, textBatch, imageBatch, conversation(map[string]any{
"type": "image_url", "image_url": map[string]any{"url": pngs(6)[5].DataURI()},
})}
require.Len(t, up.requests, len(messages), "batch texts and split six images into five and one")
for i, message := range messages {
require.Equal(t, "/v1/embeddings", up.requests[i].path)
require.Equal(t, map[string]any{
"model": name, "encoding_format": "float",
"messages": message,
}, up.requests[i].body, "no raw input, legacy truncation, or extra image prompt")
}
})
}
})
}
}
// vLLM reads matryoshka_dimensions from WeMM's config and truncates before
// normalizing. Text and image requests must use the same requested width.
func TestWeMMDimensionOverride(t *testing.T) {
for _, size := range []string{"2B", "4B", "9B"} {
t.Run(size, func(t *testing.T) {
up := newUpstream(t)
e, err := NewEmbedder(Config{
Source: types.ModelSourceRemote, Provider: "generic",
ModelName: "tencent/WeMM-Embedding-" + size, BaseURL: up.url + "/v1",
Dimensions: 1024, SupportsDimensionOverride: true,
}, nil, nil)
require.NoError(t, err)
_, err = e.Embed(types.WithEmbedQuery(context.Background()), "query")
require.NoError(t, err)
ie, ok := AsImageEmbedder(e)
require.True(t, ok)
_, err = ie.BatchEmbedImages(context.Background(), pngs(2))
require.NoError(t, err)
require.Len(t, up.requests, 2)
for _, req := range up.requests {
require.Equal(t, float64(1024), req.body["dimensions"])
require.NotContains(t, req.body, "truncate_prompt_tokens")
require.NotContains(t, req.body, "add_special_tokens")
require.NotContains(t, req.body, "add_generation_prompt")
require.NotContains(t, req.body, "use_activation")
}
})
}
}

View File

@ -107,7 +107,16 @@
"gemini/KnowledgeQA/gemini-3.7-flash": "570b30391590c3fa380e98e8323bf2187635d18db427eae3d75f9e63def26c8d",
"gemini/KnowledgeQA/gemini-3.8-flash": "6d40802ed8d3b310e1e91c186f769a4f8a298138538853cf623ad43e59c29140",
"gemini/KnowledgeQA/gemini-3snapshot": "c5931f2304774b718637f47540e78b66cbf855b4a4189c9fc0cea2040a12951d",
"generic/Embedding/WeMM-Embedding-2B": "8b6d496d4e145cebe75c192377b9a428be2b049fb23419b07e2f4ed766ee744d",
"generic/Embedding/WeMM-Embedding-4B": "d1a6ae0fbde3a9828d6392069aef8707817061b553cfc871b2f5432a460b6e83",
"generic/Embedding/WeMM-Embedding-9B": "aa6fce5495ec995031fb677c0396671b79864ab099587ac6cbc3fa52c0868e39",
"generic/Embedding/snapshot/WeMM-Embedding-2B": "4832569c35ef95ad39fc75e903fa3a860127c373e06670148301dffbaaf65294",
"generic/Embedding/snapshot/WeMM-Embedding-4B": "39c3cfdf00aacdcd443309ae6d4940a0ec16300d52aa1944460feae0174353bd",
"generic/Embedding/snapshot/WeMM-Embedding-9B": "ad2407f2bedd74df45f4cc2debe4a8f44be7c9bf138d1228c55924c477ea9a61",
"generic/Embedding/snapshotqwen3-vl-embeddingsnapshot": "a1dedaaddb23ad1449d1aef38f0f032d7bcd834bdf079c511c65ec03cccfffe4",
"generic/Embedding/tencent/WeMM-Embedding-2B": "54141d358d96ea836ce5a0ede691db0e5dff87cf7d33319a4a59d17a0c1ce0c4",
"generic/Embedding/tencent/WeMM-Embedding-4B": "0dd3bbf5caadcd0c230f5bece1aff09e6ccf6180b095c785244580936aeec68b",
"generic/Embedding/tencent/WeMM-Embedding-9B": "7176482974a360aa14a1e97dc2e7b508b3ec2526d6f38acbf7f2bf4de595b2d3",
"generic/KnowledgeQA/gpt-5snapshot": "3e3712264316db008c1465a2d7c395072d85942d0352196e456342cef0b14eb0",
"generic/KnowledgeQA/o1snapshot": "52340886b4160a0cb9342ae33a7c81304c9dad5d22c2d1c1ce073f89860a30b1",
"generic/KnowledgeQA/o3snapshot": "782e836769e56c43d9c954ab2d563df7d2f476ae4fb4d588bb8bf3c88ae42ce6",

View File

@ -0,0 +1,10 @@
# Install torch/torchvision for the target CPU/CUDA/MPS platform separately.
# Versions below match the WeMM model card and the validated service environment.
transformers==5.2.0
qwen-vl-utils==0.0.14
accelerate>=1.1.0,<2
fastapi>=0.115,<1
pydantic>=2,<3
uvicorn>=0.30,<1
pillow>=10,<13
httpx>=0.27,<1

View File

@ -0,0 +1,367 @@
#!/usr/bin/env python3
"""Serve WeMM text/image embeddings with the official Transformers model code."""
import argparse
import base64
import binascii
import hmac
import io
import logging
import os
import threading
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
from fastapi import Depends, FastAPI, Header, HTTPException
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from PIL import Image, UnidentifiedImageError
from pydantic import BaseModel, ConfigDict, Field, StrictInt
log = logging.getLogger("uvicorn.error")
@dataclass(frozen=True)
class Settings:
model: str = "tencent/WeMM-Embedding-2B"
served_model_name: str | None = None
revision: str | None = None
device: str = "auto"
dtype: str = "auto"
max_tokens: int = 8192
max_pixels: int = 1048576
max_batch_size: int = 32
cpu_threads: int = 8
local_files_only: bool = False
api_key: str = ""
def __post_init__(self):
if min(self.max_tokens, self.max_pixels, self.max_batch_size, self.cpu_threads) <= 0:
raise ValueError("Resource limits and cpu_threads must be positive.")
if self.max_pixels < 65536:
raise ValueError("max_pixels must be at least 65536 (the model processor's minimum).")
@property
def name(self):
if self.served_model_name:
return self.served_model_name
return Path(self.model).name if Path(self.model).is_dir() else self.model
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class TextPart(StrictModel):
type: Literal["text"]
text: str
class ImageURL(StrictModel):
url: str = Field(max_length=16 * 1024 * 1024)
detail: Literal["auto", "low", "high"] = "auto"
class ImagePart(StrictModel):
type: Literal["image_url"]
image_url: ImageURL
class Message(StrictModel):
role: Literal["system", "user", "assistant"]
content: str | list[TextPart | ImagePart] = Field(min_length=1)
class EmbedRequest(StrictModel):
model: str
input: str | list[str] | None = None
messages: list[Message] | list[list[Message]] | None = None
dimensions: StrictInt | None = None
encoding_format: Literal["float", "base64"] = "float"
user: str | None = None
def bad(message):
raise HTTPException(400, message)
def parse_inputs(request, settings, dimensions, default_dimension):
if request.model != settings.name:
raise HTTPException(404, f"Model not found: {request.model}")
dimension = default_dimension if request.dimensions is None else request.dimensions
if dimension not in dimensions:
bad(f"dimensions must be one of {dimensions}.")
if (request.input is None) == (request.messages is None):
bad("Supply exactly one of input or messages.")
if request.input is not None:
texts = [request.input] if isinstance(request.input, str) else request.input
conversations = [[Message(role="user", content=[TextPart(type="text", text=text)])] for text in texts]
else:
messages = request.messages
conversations = [messages] if messages and isinstance(messages[0], Message) else messages
if not conversations or len(conversations) > settings.max_batch_size:
bad(f"Supply between 1 and {settings.max_batch_size} independent inputs.")
for conversation in conversations:
if not conversation:
bad("Each conversation must contain at least one message.")
chars, images = 0, 0
for message in conversation:
parts = message.content
if isinstance(parts, str):
chars += len(parts)
continue
for part in parts:
if isinstance(part, TextPart):
chars += len(part.text)
else:
images += 1
if not part.image_url.url.startswith("data:image/") or ";base64," not in part.image_url.url:
bad("Images must be base64 data:image/... URLs; network URLs and file paths are unsupported.")
if images > 5:
bad("At most 5 images per conversation.")
if chars > settings.max_tokens * 16:
bad("Text exceeds the character guard; reduce the input size.")
return conversations, dimension
def decode_image(url):
try:
data = base64.b64decode(url.split(";base64,", 1)[1], validate=True)
with Image.open(io.BytesIO(data)) as image:
if image.width * image.height > 40_000_000:
bad("Image exceeds 40 megapixels before resizing.")
# Preserve alpha until qwen-vl-utils composites RGBA onto white,
# matching the official inference path instead of exposing hidden RGB.
return image.copy()
except (ValueError, binascii.Error, UnidentifiedImageError, OSError, Image.DecompressionBombError):
bad("Invalid image data.")
def prepare_messages(conversation, max_pixels):
messages = []
for message in conversation:
parts = message.content
if isinstance(parts, str):
parts = [TextPart(type="text", text=parts)]
content = []
for part in parts:
if isinstance(part, TextPart):
content.append({"type": "text", "text": part.text})
else:
content.append({"type": "image", "image": decode_image(part.image_url.url), "max_pixels": max_pixels})
messages.append({"role": message.role, "content": content})
return messages
def resolve_device_dtype(torch, settings):
name = settings.device
if name == "auto":
name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
device = torch.device(name)
if device.type not in ("cuda", "mps", "cpu"):
raise ValueError("device must be auto, cpu, mps, cuda or cuda:N.")
if device.type == "cuda":
if not torch.cuda.is_available():
raise ValueError("CUDA is unavailable; install a CUDA-enabled PyTorch build or select another device.")
if device.index is not None and device.index >= torch.cuda.device_count():
raise ValueError("The requested CUDA device does not exist.")
with torch.cuda.device(device):
bf16 = torch.cuda.is_bf16_supported()
elif device.type == "mps":
if not torch.backends.mps.is_available():
raise ValueError("MPS is unavailable on this machine.")
bf16 = True
else:
bf16 = False
dtype = settings.dtype
if dtype == "auto":
dtype = "float32" if device.type == "cpu" else "bfloat16" if bf16 else "float16"
if device.type == "cuda" and dtype == "bfloat16" and not bf16:
raise ValueError("This CUDA device does not support bfloat16; use --dtype float16.")
return device, getattr(torch, dtype)
class Engine:
def __init__(self, settings):
# Import after CLI setup so --mps-fallback takes effect before torch loads.
import torch
from transformers import AutoConfig, AutoModel, AutoProcessor
self.settings, self.torch = settings, torch
self.device, self.dtype = resolve_device_dtype(torch, settings)
self.lock = threading.Lock()
torch.set_num_threads(settings.cpu_threads)
options = {"trust_remote_code": True, "local_files_only": settings.local_files_only}
if settings.revision:
options["revision"] = settings.revision
config = AutoConfig.from_pretrained(settings.model, **options)
self.dimensions = tuple(getattr(config, "matryoshka_dimensions", ()))
self.default_dimension = config.text_config.hidden_size
if not self.dimensions or self.default_dimension not in self.dimensions:
raise ValueError("The model must provide WeMM matryoshka_dimensions including its hidden size.")
self.processor = AutoProcessor.from_pretrained(settings.model, **options)
self.model = AutoModel.from_pretrained(
settings.model, dtype=self.dtype, attn_implementation="sdpa", **options,
).to(self.device).eval()
if not callable(getattr(self.model, "embedding", None)):
raise ValueError("The model must expose WeMM's embedding() method.")
def encode(self, conversations, dimension):
from qwen_vl_utils import process_vision_info
torch = self.torch
vectors, total_tokens = [], 0
# WeMM's model stores rope_deltas. Keep independent conversations serialized,
# and decode images only for the current input rather than the entire batch.
with self.lock, torch.inference_mode():
for conversation in conversations:
messages = prepare_messages(conversation, self.settings.max_pixels)
text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
images, _ = process_vision_info(messages, image_patch_size=16)
inputs = self.processor(text=text, images=images, return_tensors="pt")
count = inputs["input_ids"].shape[-1]
if count > self.settings.max_tokens:
bad(f"Input has {count} tokens; limit is {self.settings.max_tokens}. Input was not truncated.")
last = self.processor.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0, -1].item())
if last != "<embedding>":
raise RuntimeError("Processor did not append the official <embedding> token.")
inputs = inputs.to(self.device)
# Reset explicitly as well, so older official code revisions are safe
# when a text request follows a multimodal request.
self.model.model.rope_deltas = None
vector = self.model.embedding(**inputs, use_cache=False).float()
vector = torch.nn.functional.normalize(vector[:, :dimension], dim=-1).cpu()[0]
if vector.numel() != dimension or not torch.isfinite(vector).all() or vector.norm() < 0.99:
raise RuntimeError("Invalid embedding output.")
vectors.append(vector)
total_tokens += count
del inputs, images, messages
return vectors, total_tokens
class BodyLimit:
"""Bound JSON bodies, including chunked requests, before parsing/base64 decoding."""
def __init__(self, app, limit=32 * 1024 * 1024):
self.app, self.limit = app, limit
async def __call__(self, scope, receive, send):
if scope["type"] != "http" or scope["method"] != "POST":
return await self.app(scope, receive, send)
chunks, size = [], 0
while True:
message = await receive()
if message["type"] == "http.disconnect":
return
chunk = message.get("body", b"")
size += len(chunk)
if size > self.limit:
response = JSONResponse({"error": {"message": "Request body exceeds 32 MiB."}}, status_code=413)
return await response(scope, receive, send)
chunks.append(chunk)
if not message.get("more_body", False):
break
body = b"".join(chunks)
async def replay():
nonlocal body
if body is not None:
value, body = body, None
return {"type": "http.request", "body": value, "more_body": False}
return await receive()
await self.app(scope, replay, send)
def create_app(settings, engine_factory=Engine):
@asynccontextmanager
async def lifespan(app):
started = time.monotonic()
app.state.engine = engine_factory(settings)
log.info("Loaded %s on %s/%s in %.1fs", settings.name, app.state.engine.device,
app.state.engine.dtype, time.monotonic() - started)
yield
del app.state.engine
app = FastAPI(title="WeMM Transformers embedding service", lifespan=lifespan)
app.add_middleware(BodyLimit)
def authorize(authorization: str | None = Header(default=None)):
expected = "Bearer " + settings.api_key
if settings.api_key and not hmac.compare_digest((authorization or "").encode(), expected.encode()):
raise HTTPException(401, "Invalid API key.", headers={"WWW-Authenticate": "Bearer"})
@app.exception_handler(HTTPException)
async def api_error(_request, exc):
return JSONResponse(status_code=exc.status_code, headers=exc.headers,
content={"error": {"message": exc.detail, "code": exc.status_code}})
@app.exception_handler(RequestValidationError)
async def validation_error(_request, exc):
return JSONResponse(status_code=400, content={"error": {
"message": "Invalid request: " + exc.errors()[0]["msg"], "code": 400,
}})
@app.get("/health")
def health():
engine = app.state.engine
return {"status": "ok", "model": settings.name, "backend": "transformers",
"device": str(engine.device), "dtype": str(engine.dtype), "revision": settings.revision,
"dimensions": engine.dimensions, "default_dimension": engine.default_dimension,
"max_tokens": settings.max_tokens, "max_image_pixels": settings.max_pixels}
@app.get("/v1/models", dependencies=[Depends(authorize)])
def models():
return {"object": "list", "data": [{"id": settings.name, "object": "model", "created": 0, "owned_by": "tencent"}]}
@app.post("/v1/embeddings", dependencies=[Depends(authorize)])
def embeddings(request: EmbedRequest):
engine = app.state.engine
conversations, dimension = parse_inputs(request, settings, engine.dimensions, engine.default_dimension)
started = time.monotonic()
try:
vectors, tokens = engine.encode(conversations, dimension)
except HTTPException:
raise
except Exception:
log.exception("WeMM inference failed")
raise HTTPException(500, "WeMM inference failed; inspect the server log.") from None
data = []
for index, vector in enumerate(vectors):
value = (base64.b64encode(vector.numpy().astype("<f4").tobytes()).decode()
if request.encoding_format == "base64" else vector.tolist())
data.append({"object": "embedding", "index": index, "embedding": value})
log.info("Embedded %d inputs, %d tokens, %d dimensions in %.2fs", len(data), tokens,
dimension, time.monotonic() - started)
return {"object": "list", "data": data, "model": settings.name,
"usage": {"prompt_tokens": tokens, "total_tokens": tokens}}
return app
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", default=Settings.model, help="WeMM Hugging Face ID or model directory")
parser.add_argument("--served-model-name", help="API model ID; defaults to the repo ID or directory basename")
parser.add_argument("--revision", help="Hugging Face revision (prefer a commit SHA for reproducibility)")
parser.add_argument("--device", default="auto", help="auto, cuda, cuda:N, mps or cpu")
parser.add_argument("--dtype", choices=("auto", "bfloat16", "float16", "float32"), default="auto")
parser.add_argument("--max-tokens", type=int, default=8192)
parser.add_argument("--max-pixels", type=int, default=1048576)
parser.add_argument("--max-batch-size", type=int, default=32)
parser.add_argument("--cpu-threads", type=int, default=8)
parser.add_argument("--local-files-only", action="store_true")
parser.add_argument("--mps-fallback", action="store_true", help="Allow PyTorch CPU fallback for unsupported MPS ops")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=18080)
args = vars(parser.parse_args())
host, port = args.pop("host"), args.pop("port")
if args.pop("mps_fallback"):
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
settings = Settings(**args, api_key=os.environ.get("WEMM_API_KEY", ""))
import uvicorn
# A single process owns the model; do not add workers without separate GPUs.
uvicorn.run(create_app(settings), host=host, port=port, workers=1)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Check a running WeMM service with actual text and image inference."""
import argparse
import base64
import io
import json
import math
import os
import time
import httpx
from PIL import Image, ImageDraw
def dot(a, b):
return sum(x * y for x, y in zip(a, b))
def text_message(text):
return [{"role": "user", "content": [{"type": "text", "text": text}]}]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:18080", help="Service root URL, without /v1")
parser.add_argument("--timeout", type=float, default=180)
args = parser.parse_args()
headers = {"Authorization": "Bearer " + os.environ["WEMM_API_KEY"]} if os.environ.get("WEMM_API_KEY") else {}
results = []
with httpx.Client(base_url=args.base_url.rstrip("/"), timeout=args.timeout, headers=headers) as client:
health_response = client.get("/health")
health_response.raise_for_status()
health = health_response.json()
model = health["model"]
default_dimension = health["default_dimension"]
response = client.get("/v1/models")
response.raise_for_status()
assert model in [row["id"] for row in response.json()["data"]]
def run(name, **payload):
started = time.monotonic()
response = client.post("/v1/embeddings", json={"model": model, **payload})
response.raise_for_status()
data = response.json()
vectors = [row["embedding"] for row in data["data"]]
assert [row["index"] for row in data["data"]] == list(range(len(vectors)))
for vector in vectors:
assert len(vector) == payload.get("dimensions", default_dimension)
assert all(math.isfinite(x) for x in vector)
assert abs(math.sqrt(dot(vector, vector)) - 1) < 1e-4
results.append({"test": name, "seconds": round(time.monotonic() - started, 3),
"vectors": len(vectors), "dimension": len(vectors[0])})
print(json.dumps(results[-1]), flush=True)
return vectors
texts = ["A red square on a white background.", "A blue circle on a black background."]
text_vectors = run("batched text messages", messages=[text_message(text) for text in texts])
images = []
for color, shape, background in [("red", "rectangle", "white"), ("blue", "ellipse", "black")]:
image = Image.new("RGB", (256, 256), background)
getattr(ImageDraw.Draw(image), shape)((40, 40, 216, 216), fill=color)
buffer = io.BytesIO()
image.save(buffer, format="PNG")
images.append([{"role": "user", "content": [{"type": "image_url", "image_url": {
"url": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode(),
}}]}])
image_vectors = run("batched image messages", messages=images)
scores = [[dot(text, image) for image in image_vectors] for text in text_vectors]
assert scores[0][0] > scores[0][1] and scores[1][1] > scores[1][0], scores
repeated = run("text after image state isolation", messages=text_message(texts[0]))[0]
assert dot(repeated, text_vectors[0]) > 0.9999
dimension = min(health["dimensions"])
short = run("Matryoshka dimensions", messages=text_message(texts[0]), dimensions=dimension)[0]
prefix = text_vectors[0][:dimension]
norm = math.sqrt(dot(prefix, prefix))
assert dot(short, [value / norm for value in prefix]) > 0.9999
raw = run("raw text compatibility", input=texts[0])[0]
assert dot(raw, text_vectors[0]) > 0.9999
for payload in ({"input": "hello", "dimensions": 3}, {"input": "a", "messages": []},
{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {
"url": "https://example.com/image.png",
}}]}]}):
response = client.post("/v1/embeddings", json={"model": model, **payload})
assert response.status_code == 400, response.text
print(json.dumps({"status": "passed", "health": health, "tests": results, "text_image_scores": scores}), flush=True)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,169 @@
"""Contract tests without downloading weights: python -m unittest discover -s scripts/wemm-transformers."""
import asyncio
import base64
import io
import struct
import unittest
from contextlib import nullcontext
from unittest.mock import patch
import torch
from fastapi import HTTPException
from fastapi.testclient import TestClient
from PIL import Image
from serve import BodyLimit, Message, Settings, create_app, decode_image, prepare_messages, resolve_device_dtype
class FakeEngine:
device, dtype = "cpu", "float32"
def __init__(self, settings, dimensions=(64, 128, 256, 512, 1024, 2048)):
self.dimensions = dimensions
self.default_dimension = dimensions[-1]
self.calls = []
def encode(self, conversations, dimension):
self.calls.append(conversations)
vectors = []
for index, _ in enumerate(conversations):
vector = torch.zeros(dimension)
vector[index] = 1
vectors.append(vector)
return vectors, len(conversations) * 10
class ServiceTests(unittest.TestCase):
def setUp(self):
self.settings = Settings()
self.engine = FakeEngine(self.settings)
self.client = self.enterContext(TestClient(create_app(self.settings, lambda _: self.engine)))
def post(self, **payload):
return self.client.post("/v1/embeddings", json={"model": self.settings.name, **payload})
def test_input_and_single_or_batched_messages(self):
message = {"role": "user", "content": [{"type": "text", "text": "hello"}]}
for payload, count in [({"input": "hello"}, 1), ({"input": ["a", "b"]}, 2),
({"messages": [message]}, 1), ({"messages": [[message], [message]]}, 2)]:
with self.subTest(payload=payload):
response = self.post(**payload)
self.assertEqual(response.status_code, 200, response.text)
data = response.json()["data"]
self.assertEqual([x["index"] for x in data], list(range(count)))
self.assertTrue(all(len(x["embedding"]) == 2048 for x in data))
self.assertEqual([x["embedding"][i] for i, x in enumerate(data)], [1] * count)
def test_all_model_sizes_use_their_own_default_and_allowed_dimensions(self):
for size, dimensions in [("2B", (64, 128, 256, 512, 1024, 2048)),
("4B", (64, 128, 256, 512, 1024, 2560)),
("9B", (64, 128, 256, 512, 1024, 2048, 4096))]:
settings = Settings(model="tencent/WeMM-Embedding-" + size)
with self.subTest(size=size), TestClient(create_app(settings, lambda s: FakeEngine(s, dimensions))) as client:
for requested in (None, 1024):
payload = {"model": settings.name, "input": "hello"}
if requested is not None:
payload["dimensions"] = requested
response = client.post("/v1/embeddings", json=payload)
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(len(response.json()["data"][0]["embedding"]), requested or dimensions[-1])
payload["dimensions"] = 2049
self.assertEqual(client.post("/v1/embeddings", json=payload).status_code, 400)
self.assertEqual(client.get("/health").json()["default_dimension"], dimensions[-1])
def test_base64_is_little_endian_float32(self):
response = self.post(input="hello", dimensions=64, encoding_format="base64")
vector = struct.unpack("<64f", base64.b64decode(response.json()["data"][0]["embedding"]))
self.assertEqual(vector, (1,) + (0,) * 63)
def test_invalid_requests_never_reach_inference(self):
for payload in [dict(input="a", messages=[]), dict(input=[]), dict(messages=[[]]),
dict(input="a", dimensions=True), dict(input="a", dimensions=0),
dict(input="a", temperature=0), dict(input=["a"] * 33),
dict(messages=[{"role": "user", "content": [{"type": "video", "video": "x"}]}]),
dict(messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]}])]:
with self.subTest(payload=payload):
self.assertEqual(self.post(**payload).status_code, 400)
self.assertEqual(self.post(model="wrong", input="a").status_code, 404)
self.assertEqual(self.engine.calls, [])
def test_authentication_on_models_and_embeddings(self):
settings = Settings(api_key="test-only-key")
with TestClient(create_app(settings, lambda _: self.engine)) as client:
self.assertEqual(client.get("/health").status_code, 200)
self.assertEqual(client.get("/v1/models").status_code, 401)
for token, status in [("Bearer wrong", 401), ("Bearer test-only-key", 200)]:
response = client.post("/v1/embeddings", json={"model": settings.name, "input": "a"},
headers={"Authorization": token})
self.assertEqual(response.status_code, status)
self.assertEqual(len(self.engine.calls), 1)
def test_image_decoding_and_invalid_bytes(self):
buffer = io.BytesIO()
Image.new("RGB", (64, 64), "red").save(buffer, format="PNG")
uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
self.assertEqual(decode_image(uri).size, (64, 64))
for data in ["!invalid!", base64.b64encode(b"not an image").decode()]:
with self.assertRaises(HTTPException) as error:
decode_image("data:image/png;base64," + data)
self.assertEqual(error.exception.status_code, 400)
def test_transparent_images_match_official_white_background(self):
from qwen_vl_utils import process_vision_info
image = Image.new("RGBA", (256, 256), (0, 0, 0, 0))
image.putpixel((128, 128), (255, 0, 0, 128))
buffer = io.BytesIO()
image.save(buffer, format="PNG")
uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
conversation = [Message.model_validate({"role": "user", "content": [
{"type": "image_url", "image_url": {"url": uri}},
]})]
actual, _ = process_vision_info(prepare_messages(conversation, 1048576), image_patch_size=16)
expected, _ = process_vision_info([{"role": "user", "content": [
{"type": "image", "image": image, "max_pixels": 1048576},
]}], image_patch_size=16)
self.assertEqual(actual[0].getpixel((0, 0)), (255, 255, 255))
self.assertEqual(actual[0].getpixel((128, 128)), (255, 127, 127))
self.assertEqual(actual[0].tobytes(), expected[0].tobytes())
def test_cpu_defaults_and_explicit_unavailable_device(self):
with patch.object(torch.cuda, "is_available", return_value=False), patch.object(torch.backends.mps, "is_available", return_value=False):
device, dtype = resolve_device_dtype(torch, Settings())
self.assertEqual(str(device), "cpu")
self.assertEqual(dtype, torch.float32)
for requested in ("cuda", "mps"):
with self.assertRaises(ValueError):
resolve_device_dtype(torch, Settings(device=requested))
def test_cuda_auto_dtype_and_explicit_bf16_guard(self):
with patch.object(torch.cuda, "is_available", return_value=True), patch.object(torch.cuda, "device", return_value=nullcontext()):
for supported, expected in [(True, torch.bfloat16), (False, torch.float16)]:
with patch.object(torch.cuda, "is_bf16_supported", return_value=supported):
device, dtype = resolve_device_dtype(torch, Settings())
self.assertEqual(str(device), "cuda")
self.assertEqual(dtype, expected)
if not supported:
with self.assertRaises(ValueError):
resolve_device_dtype(torch, Settings(device="cuda", dtype="bfloat16"))
def test_body_limit_counts_chunked_bodies(self):
async def run(chunks):
received, sent = [], []
events = iter({"type": "http.request", "body": chunk, "more_body": i < len(chunks) - 1}
for i, chunk in enumerate(chunks))
async def receive():
return next(events)
async def send(event):
sent.append(event)
async def downstream(scope, receive, send):
received.append((await receive())["body"])
await BodyLimit(downstream, limit=4)({"type": "http", "method": "POST"}, receive, send)
return received, sent
received, _ = asyncio.run(run([b"12", b"34"]))
self.assertEqual(received, [b"1234"])
received, sent = asyncio.run(run([b"12", b"345"]))
self.assertEqual(received, [])
self.assertEqual(sent[0]["status"], 413)
if __name__ == "__main__":
unittest.main()

View File

@ -95,6 +95,46 @@
| 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` |
| 自建重排服务返回的是未归一化分数(logit) | `{"score_scale": "logit"}` |
#### WeMM-Embedding
自定义厂商(`generic`)的向量模型列表已内置以下预设,选择后会自动识别图片输入和原生向量维度:
| 模型名称 | 原生维度 |
| --- | --- |
| `tencent/WeMM-Embedding-2B` | 2048 |
| `tencent/WeMM-Embedding-4B` | 2560 |
| `tencent/WeMM-Embedding-9B` | 4096 |
可选择官方 vLLM 部署,或使用本仓库的 [Transformers 部署脚本](../06-development/06-wemm-transformers.md)在 CUDA、MPS 或 CPU 上提供同一 Embedding 接口。MPS 是可选计算设备,服务也可以部署在独立服务器上。
按[官方部署说明](https://github.com/Tencent/WeMM-Embedding#serving)使用 vLLM `0.27.0`,并加载模型目录中的 `embedding_chat_template.jinja`。例如部署 2B,并通过 `--served-model-name` 让服务名称与预设一致:
```bash
MODEL_PATH=/path/to/WeMM-Embedding-2B
vllm serve "$MODEL_PATH" \
--runner pooling \
--chat-template "$MODEL_PATH/embedding_chat_template.jinja" \
--served-model-name tencent/WeMM-Embedding-2B
```
4B、9B 分别替换模型路径与服务名称。在 WeKnora 中选择「自定义」厂商,Base URL 填写服务的 `http://主机:端口/v1`,选择对应向量模型,初次接入保持「自定义输出维度」关闭。若服务没有设置 `--served-model-name`,模型名称须填写服务实际返回的本地路径;短名称 `WeMM-Embedding-2B` 及以 `/WeMM-Embedding-2B` 结尾的路径也能识别预设,4B、9B 同理。任意自定义别名需要另行配置模型能力和协议覆盖。
预设让文本和图片都通过 `messages` 进入模型模板,由模板追加取向量所需的 `<embedding>` 标记;启用 `batch_messages`,将同一批内的独立 conversations 合并请求,每条仍返回各自的向量。图片批量调用最多每批 5 张,超出自动拆分;这是 WeKnora 选择的批量大小,与现有 Qwen3-VL 预设一致,不代表模型或 vLLM 的官方上限,可用 `max_image_batch_size` 按服务承载能力调整。当前知识库图片索引任务仍逐张调用,不会因该配置自动合并多个任务。不发送历史默认的 `truncate_prompt_tokens: 511`;超出服务上下文上限时由服务报错,不会静默截断。无需额外填写兼容 JSON 或图片提示词。若此前用普通文本 `input` 为该模型建立过索引,需要重建向量。
WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_tokens: false`、`add_generation_prompt: false` 默认行为,不要照搬 Qwen3-VL 预设的 `add_special_tokens: true`。vLLM 的 Embedding 默认使用末尾 token 并做 L2 归一化,无需额外发送 `normalize` 或关闭 `use_activation`。
需要降维时,开启「自定义输出维度」并选择模型配置中的合法维度,WeKnora 会给文本和图片请求发送相同的 `dimensions`;vLLM 会先截取对应维度,再归一化:
| 模型 | 可选维度 |
| --- | --- |
| 2B | 64、128、256、512、1024、2048 |
| 4B | 64、128、256、512、1024、2560 |
| 9B | 64、128、256、512、1024、2048、4096 |
这些值来自[官方模型配置与说明](https://github.com/Tencent/WeMM-Embedding#model-zoo),vLLM 会拒绝列表之外的维度。更换输出维度后需重建已有向量索引。超时、并发与图片大小限制保持现有默认值,按实际服务性能调整。
知识库还需开启图片向量索引才能直接检索图片内容。当前预设接入文本和图片;视频、任意图文交错输入及扫描 PDF 整页向量化不在当前业务链路的支持范围内。
#### 字段参考
下表的「协议默认」是没有任何厂商、目录或覆盖时的取值;具体厂商的实际值以「实际调用方式」面板为准。

View File

@ -0,0 +1,205 @@
# WeMM Transformers 部署
使用 [WeMM 官方 Transformers 推理代码](https://github.com/Tencent/WeMM-Embedding#transformers)加载模型,由仓库内的 `scripts/wemm-transformers/serve.py` 提供 HTTP Embedding 服务。服务可以与 WeKnora 部署在同一机器,也可以部署在独立 GPU 服务器上,通过 Base URL 接入。
CUDA、Apple MPS 和 CPU 是可选的计算设备,使用同一份脚本和请求协议。此 HTTP 适配层由 WeKnora 提供,支持文本和图片;模型本身支持的视频等能力尚未在此接口开放。官方 vLLM 部署方式见[模型配置](../03-features/06-models.md#wemm-embedding)。
## 安装依赖
以下命令在 WeKnora 仓库根目录执行,使用 Python 3.12。虚拟环境和模型文件放在仓库外:
```bash
python3.12 -m venv "$HOME/.venvs/weknora-wemm"
source "$HOME/.venvs/weknora-wemm/bin/activate"
python -m pip install --upgrade pip
```
按设备选择一组 PyTorch 安装命令。以下使用已验证的 PyTorch 2.10.0 / torchvision 0.25.0,CUDA wheel 须与驱动兼容;其他 CUDA 版本可参考 [PyTorch 安装说明](https://pytorch.org/get-started/previous-versions/#v2100)。
```bash
# NVIDIA GPU:示例为 CUDA 12.8 wheel
python -m pip install torch==2.10.0 torchvision==0.25.0 \
--index-url https://download.pytorch.org/whl/cu128
# 或 Apple Silicon / MPS
python -m pip install torch==2.10.0 torchvision==0.25.0
# 或 Linux / Windows CPU wheel
python -m pip install torch==2.10.0 torchvision==0.25.0 \
--index-url https://download.pytorch.org/whl/cpu
```
安装其余依赖:
```bash
python -m pip install -r scripts/wemm-transformers/requirements.txt
```
Transformers 固定为官方推荐的 `5.2.0`,`qwen-vl-utils` 固定为 `0.0.14`。不要随意替换 Transformers 版本,避免预处理行为变化。脚本会启用 `trust_remote_code=True` 来加载 WeMM 的 `embedding()` 实现,应使用官方权重或已审查的模型目录。
## 启动服务
以 CUDA 上的 2B 为例:
```bash
python scripts/wemm-transformers/serve.py \
--model tencent/WeMM-Embedding-2B \
--revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \
--device cuda \
--host 127.0.0.1 \
--port 18080
```
首次启动会从 Hugging Face 下载模型到标准缓存,后续复用缓存。`--revision` 可固定模型代码与权重版本;示例 SHA 属于 2B,切换 4B、9B 时应选取对应仓库的版本,不能沿用此 SHA。未指定时使用该仓库默认版本。
已有模型目录时可以离线启动,并指定 WeKnora 调用时使用的模型名称:
```bash
python scripts/wemm-transformers/serve.py \
--model /srv/models/WeMM-Embedding-2B \
--served-model-name tencent/WeMM-Embedding-2B \
--local-files-only \
--device cuda
```
也可以提前下载并固定版本,例如:
```bash
hf download tencent/WeMM-Embedding-2B \
--revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \
--local-dir /srv/models/WeMM-Embedding-2B
```
其他设备只需更换参数:
```bash
# Apple Silicon,允许不支持的 MPS 算子回退到 CPU
python scripts/wemm-transformers/serve.py \
--model tencent/WeMM-Embedding-2B --device mps --mps-fallback
# CPU,默认使用 float32
python scripts/wemm-transformers/serve.py \
--model tencent/WeMM-Embedding-2B --device cpu
```
4B、9B 将 `--model` 换成对应 ID 或路径。服务从模型配置读取原生维度及 Matryoshka 可选维度,不会将其他规格强制按 2048 维处理:
| 模型 ID | 原生维度 |
| --- | --- |
| `tencent/WeMM-Embedding-2B` | 2048 |
| `tencent/WeMM-Embedding-4B` | 2560 |
| `tencent/WeMM-Embedding-9B` | 4096 |
服务以前台方式运行,按 Ctrl+C 停止。常驻部署时由 systemd、容器或其他进程管理器托管同一命令。一个进程加载一份完整模型到一个设备,不提供多 GPU 切分;选择规格前应预留权重、视觉处理和推理激活所需的显存/内存。
### 独立服务器与鉴权
跨机器部署时使用 `--host 0.0.0.0`,并在服务进程环境中设置 `WEMM_API_KEY`。`/v1/models` 与 `/v1/embeddings` 会验证 `Authorization: Bearer <key>`;WeKnora 的 API Key 填写相同值。服务不直接终止 TLS,跨不可信网络时通过 HTTPS 反向代理访问。`/health` 仅提供运行状态和非敏感模型元数据,不要求 API Key。
默认监听回环地址、未设置 API Key。WeKnora 若运行在 Docker 中,`127.0.0.1` 指向 WeKnora 容器自身,应填写容器能够访问的推理服务地址,例如同一容器网络的服务名,或已配置的宿主机地址。私网地址还需符合 WeKnora 的 SSRF 访问策略,配置方法见[安全说明](../01-getting-started/04-configuration.md)。
## WeKnora 接入
在「设置 → 模型管理」添加:
| 字段 | 填写 |
| --- | --- |
| 类型 | Embedding |
| 来源 / 服务商 | API / 自定义(OpenAI 兼容接口) |
| 模型名称 | `/v1/models` 返回的 ID,推荐保留 `tencent/WeMM-Embedding-2B` 等官方名称 |
| Base URL | `http://推理服务地址:18080/v1`,使用 HTTPS 代理时填写对应地址 |
| API Key | 设置了 `WEMM_API_KEY` 时填写相同值 |
| 图片输入 | 官方名称由预设识别;自定义别名需手动开启 |
| 自定义输出维度 | 初次接入关闭,让连接检测识别原生维度;需要降维时再开启 |
| 后台并发上限 | 建议从 1 开始 |
现有 WeMM 预设的 `messages` 格式同时适用于此服务。推荐在「高级 → 协议兼容覆盖」设置较小批量和较长超时:
```json
{
"api": "openai-embeddings",
"image_format": "messages",
"text_as_messages": true,
"batch_messages": true,
"accepts_truncate_prompt_tokens": false,
"max_batch_size": 4,
"max_image_batch_size": 2,
"request_timeout_seconds": 180
}
```
完整配置也适用于尚未包含 WeMM 预设的旧版本或自定义模型别名;此时应手动打开图片输入。点击「测试连接」,确认实际维度后保存。知识库还需开启图片向量索引,配置方式见[模型配置](../03-features/06-models.md)。切换模型或输出维度需要重建已有索引。
## 请求格式与推理流程
接口为 `POST /v1/embeddings`。纯文本可以使用 `input`:
```json
{
"model": "tencent/WeMM-Embedding-2B",
"input": ["第一段文本", "第二段文本"],
"encoding_format": "float"
}
```
文本和图片也可使用 `messages`。以下二维数组包含两个独立输入,分别返回一个文本向量和一个图片向量:
```json
{
"model": "tencent/WeMM-Embedding-2B",
"dimensions": 1024,
"messages": [
[{"role": "user", "content": [{"type": "text", "text": "白色背景上的红色方块"}]}],
[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,图片Base64"}}]}]
]
}
```
`input` 和 `messages` 只能选一个。单条 conversation 可以使用一维消息数组;文字与图片放在同一 conversation 中会得到一个联合向量。返回值采用 OpenAI Embedding 的 `data[].embedding`、`data[].index` 和 `usage` 结构,编码支持 `float` 与 float32 小端字节的 `base64`。
服务会将文本包装成消息,把内联图片解码成 PIL 图片,使用官方 processor 的聊天模板,再转换为张量。Tokenizer 自动追加 `<embedding>`,调用方无需手动添加。推理调用 `model.embedding(..., use_cache=False)`,随后按请求维度截取并做 L2 归一化。这里使用 Transformers 自带的模型 tokenizer/processor;vLLM 的 `embedding_chat_template.jinja` 不作为本脚本的额外模板传入。
HTTP 支持批量输入,但模型逐条执行,并串行处理推理请求,以隔离模型内部的可变位置状态。`batch_messages: true` 表示合并 HTTP 请求,不代表 GPU 张量批处理,也不提供 vLLM 的连续批处理调度。
## 参数与限制
| 参数 | 默认值 | 说明 |
| --- | --- | --- |
| `--device` | `auto` | 自动按 CUDA → MPS → CPU 选择;可显式指定 `cuda:N`、`mps`、`cpu`,指定不可用设备会报错 |
| `--dtype` | `auto` | CUDA 支持 BF16 时用 BF16,否则 FP16;MPS 用 BF16;CPU 用 FP32。可显式指定 `bfloat16`、`float16`、`float32` |
| `--max-tokens` | 8192 | 每个输入经过模板和图片处理后的总 token 上限,超限拒绝,不静默截断 |
| `--max-pixels` | 1048576 | 每张图片缩放后的像素上限,至少 65536;较小上限会影响高分辨率文档效果 |
| `--max-batch-size` | 32 | 每个 HTTP 请求的独立输入数上限;WeKnora 的批量设置不应超过它 |
| `--cpu-threads` | 8 | PyTorch CPU 线程数 |
| `--mps-fallback` | 关闭 | 允许 PyTorch 将不支持的 MPS 算子放到 CPU 执行 |
| `--local-files-only` | 关闭 | 仅使用本机文件或已有缓存,不下载权重 |
| `--host` / `--port` | `127.0.0.1` / `18080` | 监听地址与端口 |
每个请求体最多 32 MiB,包含分块传输;每张图片的 data URI 最多 16 MiB,解码前原图最多 4000 万像素,每个 conversation 最多 5 张图片。图片只接受内联 Base64,不抓取远程 URL 或文件路径。`image_url.detail` 仅为格式兼容,实际分辨率由 `--max-pixels` 控制。所有上限都是此部署脚本的资源设置,不代表 WeMM 模型本身的上限。
## 验证与排障
运行无需下载权重的接口测试:
```bash
python -m unittest discover -s scripts/wemm-transformers -v
```
对运行中的服务做实际图文推理测试:
```bash
python scripts/wemm-transformers/smoke_test.py \
--base-url http://127.0.0.1:18080
```
设置了鉴权时,测试进程同样读取 `WEMM_API_KEY`。测试覆盖批量图文、输出维度与归一化、图文匹配、图片后文本状态隔离及降维;输出耗时和结果到标准输出,不修改知识库。
当前在 Apple M4 Pro 上完成了 **WeMM 2B / MPS / BF16** 和 **WeMM 2B / CPU / FP32** 的完整图文推理测试。2B、4B、9B 的接口维度处理及 CUDA 精度选择有自动化覆盖;4B/9B 的完整权重推理和 CUDA 部署仍需在目标设备运行上述验证脚本。不要将接口测试视为这些设备和规格的性能验证。
常见问题:
- **CUDA/MPS 不可用**:检查 PyTorch wheel、驱动与设备,或显式改为 `--device cpu`;脚本不会将显式设备要求静默降级。
- **MPS 算子不支持**:尝试 `--mps-fallback`;BF16 不可用时可使用 `--dtype float32`,同时检查内存余量。
- **显存不足或超时**:减小模型规格、图片像素、输入长度和 WeKnora 批量。该服务为单 worker,增加 HTTP 并发不会自动增加推理吞吐。
- **模型加载缓慢**:首次需要下载权重;先预下载并固定版本,离线启动时检查目录中同时包含权重、tokenizer、processor 和官方 Python 模型代码。
- **模型名不匹配**:检查 `/v1/models`;使用自定义目录时建议显式设置 `--served-model-name`。