diff --git a/internal/models/catalog/data/models.generated.json b/internal/models/catalog/data/models.generated.json index 1851b122..59f8631a 100644 --- a/internal/models/catalog/data/models.generated.json +++ b/internal/models/catalog/data/models.generated.json @@ -2185,6 +2185,75 @@ "add_special_tokens": true } } + }, + { + "id": "tencent/WeMM-Embedding-2B", + "match": "*/WeMM-Embedding-2B", + "type": "Embedding", + "name": "WeMM-Embedding-2B", + "aliases": [ + "WeMM-Embedding-2B" + ], + "input": [ + "text", + "image" + ], + "dimension": 2048, + "source": "https://huggingface.co/tencent/WeMM-Embedding-2B", + "compat": { + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_image_batch_size": 5 + } + }, + { + "id": "tencent/WeMM-Embedding-4B", + "match": "*/WeMM-Embedding-4B", + "type": "Embedding", + "name": "WeMM-Embedding-4B", + "aliases": [ + "WeMM-Embedding-4B" + ], + "input": [ + "text", + "image" + ], + "dimension": 2560, + "source": "https://huggingface.co/tencent/WeMM-Embedding-4B", + "compat": { + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_image_batch_size": 5 + } + }, + { + "id": "tencent/WeMM-Embedding-9B", + "match": "*/WeMM-Embedding-9B", + "type": "Embedding", + "name": "WeMM-Embedding-9B", + "aliases": [ + "WeMM-Embedding-9B" + ], + "input": [ + "text", + "image" + ], + "dimension": 4096, + "source": "https://huggingface.co/tencent/WeMM-Embedding-9B", + "compat": { + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_image_batch_size": 5 + } } ], "gpustack": [], diff --git a/internal/models/catalog/data/overrides.json b/internal/models/catalog/data/overrides.json index 1123f660..fd49c67a 100644 --- a/internal/models/catalog/data/overrides.json +++ b/internal/models/catalog/data/overrides.json @@ -1050,6 +1050,45 @@ "add_special_tokens": true } } + }, + { + "id": "tencent/WeMM-Embedding-2B", + "match": "*/WeMM-Embedding-2B", + "type": "Embedding", + "compat": { + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_image_batch_size": 5 + } + }, + { + "id": "tencent/WeMM-Embedding-4B", + "match": "*/WeMM-Embedding-4B", + "type": "Embedding", + "compat": { + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_image_batch_size": 5 + } + }, + { + "id": "tencent/WeMM-Embedding-9B", + "match": "*/WeMM-Embedding-9B", + "type": "Embedding", + "compat": { + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_image_batch_size": 5 + } } ], "hunyuan": [ diff --git a/internal/models/catalog/data/seed.json b/internal/models/catalog/data/seed.json index a2ca1639..02b4b0c5 100644 --- a/internal/models/catalog/data/seed.json +++ b/internal/models/catalog/data/seed.json @@ -1454,6 +1454,51 @@ "image" ], "source": "https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B" + }, + { + "id": "tencent/WeMM-Embedding-2B", + "match": "*/WeMM-Embedding-2B", + "type": "Embedding", + "name": "WeMM-Embedding-2B", + "aliases": [ + "WeMM-Embedding-2B" + ], + "input": [ + "text", + "image" + ], + "dimension": 2048, + "source": "https://huggingface.co/tencent/WeMM-Embedding-2B" + }, + { + "id": "tencent/WeMM-Embedding-4B", + "match": "*/WeMM-Embedding-4B", + "type": "Embedding", + "name": "WeMM-Embedding-4B", + "aliases": [ + "WeMM-Embedding-4B" + ], + "input": [ + "text", + "image" + ], + "dimension": 2560, + "source": "https://huggingface.co/tencent/WeMM-Embedding-4B" + }, + { + "id": "tencent/WeMM-Embedding-9B", + "match": "*/WeMM-Embedding-9B", + "type": "Embedding", + "name": "WeMM-Embedding-9B", + "aliases": [ + "WeMM-Embedding-9B" + ], + "input": [ + "text", + "image" + ], + "dimension": 4096, + "source": "https://huggingface.co/tencent/WeMM-Embedding-9B" } ], "gpustack": [], diff --git a/internal/models/embedding/wemm_test.go b/internal/models/embedding/wemm_test.go new file mode 100644 index 00000000..012c2b79 --- /dev/null +++ b/internal/models/embedding/wemm_test.go @@ -0,0 +1,127 @@ +package embedding + +import ( + "context" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + modelruntime "github.com/Tencent/WeKnora/internal/models/runtime" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +// WeMM's vLLM template appends the dedicated pooling token. +// Both retrieval sides and images must use that template, including when +// vLLM exposes the local model path instead of the Hugging Face repository ID. +// https://huggingface.co/tencent/WeMM-Embedding-2B/blob/main/embedding_chat_template.jinja +func TestWeMMVLLMPresets(t *testing.T) { + vendor, ok := modelruntime.Get("generic") + require.True(t, ok) + listed := vendor.ModelsByType(types.ModelTypeEmbedding) + for _, tc := range []struct { + size string + dims int + }{{"2B", 2048}, {"4B", 2560}, {"9B", 4096}} { + short := "WeMM-Embedding-" + tc.size + id := "tencent/" + short + t.Run(tc.size, func(t *testing.T) { + visible := false + for _, entry := range listed { + if entry.ID == id { + visible = true + require.Equal(t, tc.dims, entry.Dimension) + } + } + require.True(t, visible, "preset must appear in the embedding picker") + for _, name := range []string{id, short, "/models/" + short, strings.ToLower(id)} { + t.Run(name, func(t *testing.T) { + resolved, err := modelruntime.Resolve(modelruntime.Ref{ + Provider: "generic", Model: name, ModelType: types.ModelTypeEmbedding, + }) + require.NoError(t, err) + require.True(t, resolved.Cataloged) + require.Equal(t, tc.dims, resolved.Spec.Dimension) + require.Equal(t, api.EmbeddingOpenAI, resolved.EmbeddingAPI) + require.Equal(t, []string{"text", "image"}, resolved.Spec.Input) + require.True(t, resolved.Embeddings.BatchMessages) + require.Equal(t, 5, resolved.Embeddings.ImageBatchLimit()) + + up := newUpstream(t) + e, err := NewEmbedder(Config{ + Source: types.ModelSourceRemote, Provider: "generic", ModelName: name, + BaseURL: up.url + "/v1", Dimensions: tc.dims, TruncatePromptTokens: 511, + }, nil, nil) + require.NoError(t, err) + require.Equal(t, tc.dims, e.GetDimensions()) + ie, ok := AsImageEmbedder(e) + require.True(t, ok, "images must work without a row override") + contexts := []context.Context{context.Background(), types.WithEmbedQuery(context.Background())} + for _, ctx := range contexts { + vectors, err := e.BatchEmbed(ctx, []string{"ab", "c"}) + require.NoError(t, err) + require.Equal(t, [][]float32{{2}, {1}}, vectors) + } + vectors, err := ie.BatchEmbedImages(context.Background(), pngs(6)) + require.NoError(t, err) + require.Equal(t, [][]float32{{1}, {2}, {3}, {4}, {5}, {6}}, vectors) + + conversation := func(part map[string]any) []any { + return []any{map[string]any{"role": "user", "content": []any{part}}} + } + textBatch := []any{ + conversation(map[string]any{"type": "text", "text": "ab"}), + conversation(map[string]any{"type": "text", "text": "c"}), + } + imageBatch := []any{} + for _, img := range pngs(5) { + imageBatch = append(imageBatch, conversation(map[string]any{ + "type": "image_url", "image_url": map[string]any{"url": img.DataURI()}, + })) + } + messages := []any{textBatch, textBatch, imageBatch, conversation(map[string]any{ + "type": "image_url", "image_url": map[string]any{"url": pngs(6)[5].DataURI()}, + })} + require.Len(t, up.requests, len(messages), "batch texts and split six images into five and one") + for i, message := range messages { + require.Equal(t, "/v1/embeddings", up.requests[i].path) + require.Equal(t, map[string]any{ + "model": name, "encoding_format": "float", + "messages": message, + }, up.requests[i].body, "no raw input, legacy truncation, or extra image prompt") + } + }) + } + }) + } +} + +// vLLM reads matryoshka_dimensions from WeMM's config and truncates before +// normalizing. Text and image requests must use the same requested width. +func TestWeMMDimensionOverride(t *testing.T) { + for _, size := range []string{"2B", "4B", "9B"} { + t.Run(size, func(t *testing.T) { + up := newUpstream(t) + e, err := NewEmbedder(Config{ + Source: types.ModelSourceRemote, Provider: "generic", + ModelName: "tencent/WeMM-Embedding-" + size, BaseURL: up.url + "/v1", + Dimensions: 1024, SupportsDimensionOverride: true, + }, nil, nil) + require.NoError(t, err) + _, err = e.Embed(types.WithEmbedQuery(context.Background()), "query") + require.NoError(t, err) + ie, ok := AsImageEmbedder(e) + require.True(t, ok) + _, err = ie.BatchEmbedImages(context.Background(), pngs(2)) + require.NoError(t, err) + require.Len(t, up.requests, 2) + for _, req := range up.requests { + require.Equal(t, float64(1024), req.body["dimensions"]) + require.NotContains(t, req.body, "truncate_prompt_tokens") + require.NotContains(t, req.body, "add_special_tokens") + require.NotContains(t, req.body, "add_generation_prompt") + require.NotContains(t, req.body, "use_activation") + } + }) + } +} diff --git a/internal/models/parity/testdata/catalog-baseline.json b/internal/models/parity/testdata/catalog-baseline.json index 42eae44a..2e293620 100644 --- a/internal/models/parity/testdata/catalog-baseline.json +++ b/internal/models/parity/testdata/catalog-baseline.json @@ -107,7 +107,16 @@ "gemini/KnowledgeQA/gemini-3.7-flash": "570b30391590c3fa380e98e8323bf2187635d18db427eae3d75f9e63def26c8d", "gemini/KnowledgeQA/gemini-3.8-flash": "6d40802ed8d3b310e1e91c186f769a4f8a298138538853cf623ad43e59c29140", "gemini/KnowledgeQA/gemini-3snapshot": "c5931f2304774b718637f47540e78b66cbf855b4a4189c9fc0cea2040a12951d", + "generic/Embedding/WeMM-Embedding-2B": "8b6d496d4e145cebe75c192377b9a428be2b049fb23419b07e2f4ed766ee744d", + "generic/Embedding/WeMM-Embedding-4B": "d1a6ae0fbde3a9828d6392069aef8707817061b553cfc871b2f5432a460b6e83", + "generic/Embedding/WeMM-Embedding-9B": "aa6fce5495ec995031fb677c0396671b79864ab099587ac6cbc3fa52c0868e39", + "generic/Embedding/snapshot/WeMM-Embedding-2B": "4832569c35ef95ad39fc75e903fa3a860127c373e06670148301dffbaaf65294", + "generic/Embedding/snapshot/WeMM-Embedding-4B": "39c3cfdf00aacdcd443309ae6d4940a0ec16300d52aa1944460feae0174353bd", + "generic/Embedding/snapshot/WeMM-Embedding-9B": "ad2407f2bedd74df45f4cc2debe4a8f44be7c9bf138d1228c55924c477ea9a61", "generic/Embedding/snapshotqwen3-vl-embeddingsnapshot": "a1dedaaddb23ad1449d1aef38f0f032d7bcd834bdf079c511c65ec03cccfffe4", + "generic/Embedding/tencent/WeMM-Embedding-2B": "54141d358d96ea836ce5a0ede691db0e5dff87cf7d33319a4a59d17a0c1ce0c4", + "generic/Embedding/tencent/WeMM-Embedding-4B": "0dd3bbf5caadcd0c230f5bece1aff09e6ccf6180b095c785244580936aeec68b", + "generic/Embedding/tencent/WeMM-Embedding-9B": "7176482974a360aa14a1e97dc2e7b508b3ec2526d6f38acbf7f2bf4de595b2d3", "generic/KnowledgeQA/gpt-5snapshot": "3e3712264316db008c1465a2d7c395072d85942d0352196e456342cef0b14eb0", "generic/KnowledgeQA/o1snapshot": "52340886b4160a0cb9342ae33a7c81304c9dad5d22c2d1c1ce073f89860a30b1", "generic/KnowledgeQA/o3snapshot": "782e836769e56c43d9c954ab2d563df7d2f476ae4fb4d588bb8bf3c88ae42ce6", diff --git a/scripts/wemm-transformers/requirements.txt b/scripts/wemm-transformers/requirements.txt new file mode 100644 index 00000000..c9a8dd40 --- /dev/null +++ b/scripts/wemm-transformers/requirements.txt @@ -0,0 +1,10 @@ +# Install torch/torchvision for the target CPU/CUDA/MPS platform separately. +# Versions below match the WeMM model card and the validated service environment. +transformers==5.2.0 +qwen-vl-utils==0.0.14 +accelerate>=1.1.0,<2 +fastapi>=0.115,<1 +pydantic>=2,<3 +uvicorn>=0.30,<1 +pillow>=10,<13 +httpx>=0.27,<1 diff --git a/scripts/wemm-transformers/serve.py b/scripts/wemm-transformers/serve.py new file mode 100644 index 00000000..08b36ab3 --- /dev/null +++ b/scripts/wemm-transformers/serve.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Serve WeMM text/image embeddings with the official Transformers model code.""" +import argparse +import base64 +import binascii +import hmac +import io +import logging +import os +import threading +import time +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from fastapi import Depends, FastAPI, Header, HTTPException +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from PIL import Image, UnidentifiedImageError +from pydantic import BaseModel, ConfigDict, Field, StrictInt + +log = logging.getLogger("uvicorn.error") + + +@dataclass(frozen=True) +class Settings: + model: str = "tencent/WeMM-Embedding-2B" + served_model_name: str | None = None + revision: str | None = None + device: str = "auto" + dtype: str = "auto" + max_tokens: int = 8192 + max_pixels: int = 1048576 + max_batch_size: int = 32 + cpu_threads: int = 8 + local_files_only: bool = False + api_key: str = "" + + def __post_init__(self): + if min(self.max_tokens, self.max_pixels, self.max_batch_size, self.cpu_threads) <= 0: + raise ValueError("Resource limits and cpu_threads must be positive.") + if self.max_pixels < 65536: + raise ValueError("max_pixels must be at least 65536 (the model processor's minimum).") + + @property + def name(self): + if self.served_model_name: + return self.served_model_name + return Path(self.model).name if Path(self.model).is_dir() else self.model + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class TextPart(StrictModel): + type: Literal["text"] + text: str + + +class ImageURL(StrictModel): + url: str = Field(max_length=16 * 1024 * 1024) + detail: Literal["auto", "low", "high"] = "auto" + + +class ImagePart(StrictModel): + type: Literal["image_url"] + image_url: ImageURL + + +class Message(StrictModel): + role: Literal["system", "user", "assistant"] + content: str | list[TextPart | ImagePart] = Field(min_length=1) + + +class EmbedRequest(StrictModel): + model: str + input: str | list[str] | None = None + messages: list[Message] | list[list[Message]] | None = None + dimensions: StrictInt | None = None + encoding_format: Literal["float", "base64"] = "float" + user: str | None = None + + +def bad(message): + raise HTTPException(400, message) + + +def parse_inputs(request, settings, dimensions, default_dimension): + if request.model != settings.name: + raise HTTPException(404, f"Model not found: {request.model}") + dimension = default_dimension if request.dimensions is None else request.dimensions + if dimension not in dimensions: + bad(f"dimensions must be one of {dimensions}.") + if (request.input is None) == (request.messages is None): + bad("Supply exactly one of input or messages.") + if request.input is not None: + texts = [request.input] if isinstance(request.input, str) else request.input + conversations = [[Message(role="user", content=[TextPart(type="text", text=text)])] for text in texts] + else: + messages = request.messages + conversations = [messages] if messages and isinstance(messages[0], Message) else messages + if not conversations or len(conversations) > settings.max_batch_size: + bad(f"Supply between 1 and {settings.max_batch_size} independent inputs.") + for conversation in conversations: + if not conversation: + bad("Each conversation must contain at least one message.") + chars, images = 0, 0 + for message in conversation: + parts = message.content + if isinstance(parts, str): + chars += len(parts) + continue + for part in parts: + if isinstance(part, TextPart): + chars += len(part.text) + else: + images += 1 + if not part.image_url.url.startswith("data:image/") or ";base64," not in part.image_url.url: + bad("Images must be base64 data:image/... URLs; network URLs and file paths are unsupported.") + if images > 5: + bad("At most 5 images per conversation.") + if chars > settings.max_tokens * 16: + bad("Text exceeds the character guard; reduce the input size.") + return conversations, dimension + + +def decode_image(url): + try: + data = base64.b64decode(url.split(";base64,", 1)[1], validate=True) + with Image.open(io.BytesIO(data)) as image: + if image.width * image.height > 40_000_000: + bad("Image exceeds 40 megapixels before resizing.") + # Preserve alpha until qwen-vl-utils composites RGBA onto white, + # matching the official inference path instead of exposing hidden RGB. + return image.copy() + except (ValueError, binascii.Error, UnidentifiedImageError, OSError, Image.DecompressionBombError): + bad("Invalid image data.") + + +def prepare_messages(conversation, max_pixels): + messages = [] + for message in conversation: + parts = message.content + if isinstance(parts, str): + parts = [TextPart(type="text", text=parts)] + content = [] + for part in parts: + if isinstance(part, TextPart): + content.append({"type": "text", "text": part.text}) + else: + content.append({"type": "image", "image": decode_image(part.image_url.url), "max_pixels": max_pixels}) + messages.append({"role": message.role, "content": content}) + return messages + + +def resolve_device_dtype(torch, settings): + name = settings.device + if name == "auto": + name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" + device = torch.device(name) + if device.type not in ("cuda", "mps", "cpu"): + raise ValueError("device must be auto, cpu, mps, cuda or cuda:N.") + if device.type == "cuda": + if not torch.cuda.is_available(): + raise ValueError("CUDA is unavailable; install a CUDA-enabled PyTorch build or select another device.") + if device.index is not None and device.index >= torch.cuda.device_count(): + raise ValueError("The requested CUDA device does not exist.") + with torch.cuda.device(device): + bf16 = torch.cuda.is_bf16_supported() + elif device.type == "mps": + if not torch.backends.mps.is_available(): + raise ValueError("MPS is unavailable on this machine.") + bf16 = True + else: + bf16 = False + dtype = settings.dtype + if dtype == "auto": + dtype = "float32" if device.type == "cpu" else "bfloat16" if bf16 else "float16" + if device.type == "cuda" and dtype == "bfloat16" and not bf16: + raise ValueError("This CUDA device does not support bfloat16; use --dtype float16.") + return device, getattr(torch, dtype) + + +class Engine: + def __init__(self, settings): + # Import after CLI setup so --mps-fallback takes effect before torch loads. + import torch + from transformers import AutoConfig, AutoModel, AutoProcessor + self.settings, self.torch = settings, torch + self.device, self.dtype = resolve_device_dtype(torch, settings) + self.lock = threading.Lock() + torch.set_num_threads(settings.cpu_threads) + options = {"trust_remote_code": True, "local_files_only": settings.local_files_only} + if settings.revision: + options["revision"] = settings.revision + config = AutoConfig.from_pretrained(settings.model, **options) + self.dimensions = tuple(getattr(config, "matryoshka_dimensions", ())) + self.default_dimension = config.text_config.hidden_size + if not self.dimensions or self.default_dimension not in self.dimensions: + raise ValueError("The model must provide WeMM matryoshka_dimensions including its hidden size.") + self.processor = AutoProcessor.from_pretrained(settings.model, **options) + self.model = AutoModel.from_pretrained( + settings.model, dtype=self.dtype, attn_implementation="sdpa", **options, + ).to(self.device).eval() + if not callable(getattr(self.model, "embedding", None)): + raise ValueError("The model must expose WeMM's embedding() method.") + + def encode(self, conversations, dimension): + from qwen_vl_utils import process_vision_info + torch = self.torch + vectors, total_tokens = [], 0 + # WeMM's model stores rope_deltas. Keep independent conversations serialized, + # and decode images only for the current input rather than the entire batch. + with self.lock, torch.inference_mode(): + for conversation in conversations: + messages = prepare_messages(conversation, self.settings.max_pixels) + text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False) + images, _ = process_vision_info(messages, image_patch_size=16) + inputs = self.processor(text=text, images=images, return_tensors="pt") + count = inputs["input_ids"].shape[-1] + if count > self.settings.max_tokens: + bad(f"Input has {count} tokens; limit is {self.settings.max_tokens}. Input was not truncated.") + last = self.processor.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0, -1].item()) + if last != "": + raise RuntimeError("Processor did not append the official token.") + inputs = inputs.to(self.device) + # Reset explicitly as well, so older official code revisions are safe + # when a text request follows a multimodal request. + self.model.model.rope_deltas = None + vector = self.model.embedding(**inputs, use_cache=False).float() + vector = torch.nn.functional.normalize(vector[:, :dimension], dim=-1).cpu()[0] + if vector.numel() != dimension or not torch.isfinite(vector).all() or vector.norm() < 0.99: + raise RuntimeError("Invalid embedding output.") + vectors.append(vector) + total_tokens += count + del inputs, images, messages + return vectors, total_tokens + + +class BodyLimit: + """Bound JSON bodies, including chunked requests, before parsing/base64 decoding.""" + def __init__(self, app, limit=32 * 1024 * 1024): + self.app, self.limit = app, limit + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or scope["method"] != "POST": + return await self.app(scope, receive, send) + chunks, size = [], 0 + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + chunk = message.get("body", b"") + size += len(chunk) + if size > self.limit: + response = JSONResponse({"error": {"message": "Request body exceeds 32 MiB."}}, status_code=413) + return await response(scope, receive, send) + chunks.append(chunk) + if not message.get("more_body", False): + break + body = b"".join(chunks) + + async def replay(): + nonlocal body + if body is not None: + value, body = body, None + return {"type": "http.request", "body": value, "more_body": False} + return await receive() + + await self.app(scope, replay, send) + + +def create_app(settings, engine_factory=Engine): + @asynccontextmanager + async def lifespan(app): + started = time.monotonic() + app.state.engine = engine_factory(settings) + log.info("Loaded %s on %s/%s in %.1fs", settings.name, app.state.engine.device, + app.state.engine.dtype, time.monotonic() - started) + yield + del app.state.engine + + app = FastAPI(title="WeMM Transformers embedding service", lifespan=lifespan) + app.add_middleware(BodyLimit) + + def authorize(authorization: str | None = Header(default=None)): + expected = "Bearer " + settings.api_key + if settings.api_key and not hmac.compare_digest((authorization or "").encode(), expected.encode()): + raise HTTPException(401, "Invalid API key.", headers={"WWW-Authenticate": "Bearer"}) + + @app.exception_handler(HTTPException) + async def api_error(_request, exc): + return JSONResponse(status_code=exc.status_code, headers=exc.headers, + content={"error": {"message": exc.detail, "code": exc.status_code}}) + + @app.exception_handler(RequestValidationError) + async def validation_error(_request, exc): + return JSONResponse(status_code=400, content={"error": { + "message": "Invalid request: " + exc.errors()[0]["msg"], "code": 400, + }}) + + @app.get("/health") + def health(): + engine = app.state.engine + return {"status": "ok", "model": settings.name, "backend": "transformers", + "device": str(engine.device), "dtype": str(engine.dtype), "revision": settings.revision, + "dimensions": engine.dimensions, "default_dimension": engine.default_dimension, + "max_tokens": settings.max_tokens, "max_image_pixels": settings.max_pixels} + + @app.get("/v1/models", dependencies=[Depends(authorize)]) + def models(): + return {"object": "list", "data": [{"id": settings.name, "object": "model", "created": 0, "owned_by": "tencent"}]} + + @app.post("/v1/embeddings", dependencies=[Depends(authorize)]) + def embeddings(request: EmbedRequest): + engine = app.state.engine + conversations, dimension = parse_inputs(request, settings, engine.dimensions, engine.default_dimension) + started = time.monotonic() + try: + vectors, tokens = engine.encode(conversations, dimension) + except HTTPException: + raise + except Exception: + log.exception("WeMM inference failed") + raise HTTPException(500, "WeMM inference failed; inspect the server log.") from None + data = [] + for index, vector in enumerate(vectors): + value = (base64.b64encode(vector.numpy().astype(" scores[0][1] and scores[1][1] > scores[1][0], scores + repeated = run("text after image state isolation", messages=text_message(texts[0]))[0] + assert dot(repeated, text_vectors[0]) > 0.9999 + dimension = min(health["dimensions"]) + short = run("Matryoshka dimensions", messages=text_message(texts[0]), dimensions=dimension)[0] + prefix = text_vectors[0][:dimension] + norm = math.sqrt(dot(prefix, prefix)) + assert dot(short, [value / norm for value in prefix]) > 0.9999 + raw = run("raw text compatibility", input=texts[0])[0] + assert dot(raw, text_vectors[0]) > 0.9999 + for payload in ({"input": "hello", "dimensions": 3}, {"input": "a", "messages": []}, + {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": { + "url": "https://example.com/image.png", + }}]}]}): + response = client.post("/v1/embeddings", json={"model": model, **payload}) + assert response.status_code == 400, response.text + print(json.dumps({"status": "passed", "health": health, "tests": results, "text_image_scores": scores}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/wemm-transformers/test_serve.py b/scripts/wemm-transformers/test_serve.py new file mode 100644 index 00000000..e08f5a53 --- /dev/null +++ b/scripts/wemm-transformers/test_serve.py @@ -0,0 +1,169 @@ +"""Contract tests without downloading weights: python -m unittest discover -s scripts/wemm-transformers.""" +import asyncio +import base64 +import io +import struct +import unittest +from contextlib import nullcontext +from unittest.mock import patch + +import torch +from fastapi import HTTPException +from fastapi.testclient import TestClient +from PIL import Image +from serve import BodyLimit, Message, Settings, create_app, decode_image, prepare_messages, resolve_device_dtype + + +class FakeEngine: + device, dtype = "cpu", "float32" + + def __init__(self, settings, dimensions=(64, 128, 256, 512, 1024, 2048)): + self.dimensions = dimensions + self.default_dimension = dimensions[-1] + self.calls = [] + + def encode(self, conversations, dimension): + self.calls.append(conversations) + vectors = [] + for index, _ in enumerate(conversations): + vector = torch.zeros(dimension) + vector[index] = 1 + vectors.append(vector) + return vectors, len(conversations) * 10 + + +class ServiceTests(unittest.TestCase): + def setUp(self): + self.settings = Settings() + self.engine = FakeEngine(self.settings) + self.client = self.enterContext(TestClient(create_app(self.settings, lambda _: self.engine))) + + def post(self, **payload): + return self.client.post("/v1/embeddings", json={"model": self.settings.name, **payload}) + + def test_input_and_single_or_batched_messages(self): + message = {"role": "user", "content": [{"type": "text", "text": "hello"}]} + for payload, count in [({"input": "hello"}, 1), ({"input": ["a", "b"]}, 2), + ({"messages": [message]}, 1), ({"messages": [[message], [message]]}, 2)]: + with self.subTest(payload=payload): + response = self.post(**payload) + self.assertEqual(response.status_code, 200, response.text) + data = response.json()["data"] + self.assertEqual([x["index"] for x in data], list(range(count))) + self.assertTrue(all(len(x["embedding"]) == 2048 for x in data)) + self.assertEqual([x["embedding"][i] for i, x in enumerate(data)], [1] * count) + + def test_all_model_sizes_use_their_own_default_and_allowed_dimensions(self): + for size, dimensions in [("2B", (64, 128, 256, 512, 1024, 2048)), + ("4B", (64, 128, 256, 512, 1024, 2560)), + ("9B", (64, 128, 256, 512, 1024, 2048, 4096))]: + settings = Settings(model="tencent/WeMM-Embedding-" + size) + with self.subTest(size=size), TestClient(create_app(settings, lambda s: FakeEngine(s, dimensions))) as client: + for requested in (None, 1024): + payload = {"model": settings.name, "input": "hello"} + if requested is not None: + payload["dimensions"] = requested + response = client.post("/v1/embeddings", json=payload) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(response.json()["data"][0]["embedding"]), requested or dimensions[-1]) + payload["dimensions"] = 2049 + self.assertEqual(client.post("/v1/embeddings", json=payload).status_code, 400) + self.assertEqual(client.get("/health").json()["default_dimension"], dimensions[-1]) + + def test_base64_is_little_endian_float32(self): + response = self.post(input="hello", dimensions=64, encoding_format="base64") + vector = struct.unpack("<64f", base64.b64decode(response.json()["data"][0]["embedding"])) + self.assertEqual(vector, (1,) + (0,) * 63) + + def test_invalid_requests_never_reach_inference(self): + for payload in [dict(input="a", messages=[]), dict(input=[]), dict(messages=[[]]), + dict(input="a", dimensions=True), dict(input="a", dimensions=0), + dict(input="a", temperature=0), dict(input=["a"] * 33), + dict(messages=[{"role": "user", "content": [{"type": "video", "video": "x"}]}]), + dict(messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]}])]: + with self.subTest(payload=payload): + self.assertEqual(self.post(**payload).status_code, 400) + self.assertEqual(self.post(model="wrong", input="a").status_code, 404) + self.assertEqual(self.engine.calls, []) + + def test_authentication_on_models_and_embeddings(self): + settings = Settings(api_key="test-only-key") + with TestClient(create_app(settings, lambda _: self.engine)) as client: + self.assertEqual(client.get("/health").status_code, 200) + self.assertEqual(client.get("/v1/models").status_code, 401) + for token, status in [("Bearer wrong", 401), ("Bearer test-only-key", 200)]: + response = client.post("/v1/embeddings", json={"model": settings.name, "input": "a"}, + headers={"Authorization": token}) + self.assertEqual(response.status_code, status) + self.assertEqual(len(self.engine.calls), 1) + + def test_image_decoding_and_invalid_bytes(self): + buffer = io.BytesIO() + Image.new("RGB", (64, 64), "red").save(buffer, format="PNG") + uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + self.assertEqual(decode_image(uri).size, (64, 64)) + for data in ["!invalid!", base64.b64encode(b"not an image").decode()]: + with self.assertRaises(HTTPException) as error: + decode_image("data:image/png;base64," + data) + self.assertEqual(error.exception.status_code, 400) + + def test_transparent_images_match_official_white_background(self): + from qwen_vl_utils import process_vision_info + image = Image.new("RGBA", (256, 256), (0, 0, 0, 0)) + image.putpixel((128, 128), (255, 0, 0, 128)) + buffer = io.BytesIO() + image.save(buffer, format="PNG") + uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + conversation = [Message.model_validate({"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": uri}}, + ]})] + actual, _ = process_vision_info(prepare_messages(conversation, 1048576), image_patch_size=16) + expected, _ = process_vision_info([{"role": "user", "content": [ + {"type": "image", "image": image, "max_pixels": 1048576}, + ]}], image_patch_size=16) + self.assertEqual(actual[0].getpixel((0, 0)), (255, 255, 255)) + self.assertEqual(actual[0].getpixel((128, 128)), (255, 127, 127)) + self.assertEqual(actual[0].tobytes(), expected[0].tobytes()) + + def test_cpu_defaults_and_explicit_unavailable_device(self): + with patch.object(torch.cuda, "is_available", return_value=False), patch.object(torch.backends.mps, "is_available", return_value=False): + device, dtype = resolve_device_dtype(torch, Settings()) + self.assertEqual(str(device), "cpu") + self.assertEqual(dtype, torch.float32) + for requested in ("cuda", "mps"): + with self.assertRaises(ValueError): + resolve_device_dtype(torch, Settings(device=requested)) + + def test_cuda_auto_dtype_and_explicit_bf16_guard(self): + with patch.object(torch.cuda, "is_available", return_value=True), patch.object(torch.cuda, "device", return_value=nullcontext()): + for supported, expected in [(True, torch.bfloat16), (False, torch.float16)]: + with patch.object(torch.cuda, "is_bf16_supported", return_value=supported): + device, dtype = resolve_device_dtype(torch, Settings()) + self.assertEqual(str(device), "cuda") + self.assertEqual(dtype, expected) + if not supported: + with self.assertRaises(ValueError): + resolve_device_dtype(torch, Settings(device="cuda", dtype="bfloat16")) + + def test_body_limit_counts_chunked_bodies(self): + async def run(chunks): + received, sent = [], [] + events = iter({"type": "http.request", "body": chunk, "more_body": i < len(chunks) - 1} + for i, chunk in enumerate(chunks)) + async def receive(): + return next(events) + async def send(event): + sent.append(event) + async def downstream(scope, receive, send): + received.append((await receive())["body"]) + await BodyLimit(downstream, limit=4)({"type": "http", "method": "POST"}, receive, send) + return received, sent + received, _ = asyncio.run(run([b"12", b"34"])) + self.assertEqual(received, [b"1234"]) + received, sent = asyncio.run(run([b"12", b"345"])) + self.assertEqual(received, []) + self.assertEqual(sent[0]["status"], 413) + + +if __name__ == "__main__": + unittest.main() diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.md index ea8ddb2e..68cd82a2 100644 --- a/website-docs/03-features/06-models.md +++ b/website-docs/03-features/06-models.md @@ -95,6 +95,46 @@ | 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` | | 自建重排服务返回的是未归一化分数(logit) | `{"score_scale": "logit"}` | +#### WeMM-Embedding + +自定义厂商(`generic`)的向量模型列表已内置以下预设,选择后会自动识别图片输入和原生向量维度: + +| 模型名称 | 原生维度 | +| --- | --- | +| `tencent/WeMM-Embedding-2B` | 2048 | +| `tencent/WeMM-Embedding-4B` | 2560 | +| `tencent/WeMM-Embedding-9B` | 4096 | + +可选择官方 vLLM 部署,或使用本仓库的 [Transformers 部署脚本](../06-development/06-wemm-transformers.md)在 CUDA、MPS 或 CPU 上提供同一 Embedding 接口。MPS 是可选计算设备,服务也可以部署在独立服务器上。 + +按[官方部署说明](https://github.com/Tencent/WeMM-Embedding#serving)使用 vLLM `0.27.0`,并加载模型目录中的 `embedding_chat_template.jinja`。例如部署 2B,并通过 `--served-model-name` 让服务名称与预设一致: + +```bash +MODEL_PATH=/path/to/WeMM-Embedding-2B +vllm serve "$MODEL_PATH" \ + --runner pooling \ + --chat-template "$MODEL_PATH/embedding_chat_template.jinja" \ + --served-model-name tencent/WeMM-Embedding-2B +``` + +4B、9B 分别替换模型路径与服务名称。在 WeKnora 中选择「自定义」厂商,Base URL 填写服务的 `http://主机:端口/v1`,选择对应向量模型,初次接入保持「自定义输出维度」关闭。若服务没有设置 `--served-model-name`,模型名称须填写服务实际返回的本地路径;短名称 `WeMM-Embedding-2B` 及以 `/WeMM-Embedding-2B` 结尾的路径也能识别预设,4B、9B 同理。任意自定义别名需要另行配置模型能力和协议覆盖。 + +预设让文本和图片都通过 `messages` 进入模型模板,由模板追加取向量所需的 `` 标记;启用 `batch_messages`,将同一批内的独立 conversations 合并请求,每条仍返回各自的向量。图片批量调用最多每批 5 张,超出自动拆分;这是 WeKnora 选择的批量大小,与现有 Qwen3-VL 预设一致,不代表模型或 vLLM 的官方上限,可用 `max_image_batch_size` 按服务承载能力调整。当前知识库图片索引任务仍逐张调用,不会因该配置自动合并多个任务。不发送历史默认的 `truncate_prompt_tokens: 511`;超出服务上下文上限时由服务报错,不会静默截断。无需额外填写兼容 JSON 或图片提示词。若此前用普通文本 `input` 为该模型建立过索引,需要重建向量。 + +WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_tokens: false`、`add_generation_prompt: false` 默认行为,不要照搬 Qwen3-VL 预设的 `add_special_tokens: true`。vLLM 的 Embedding 默认使用末尾 token 并做 L2 归一化,无需额外发送 `normalize` 或关闭 `use_activation`。 + +需要降维时,开启「自定义输出维度」并选择模型配置中的合法维度,WeKnora 会给文本和图片请求发送相同的 `dimensions`;vLLM 会先截取对应维度,再归一化: + +| 模型 | 可选维度 | +| --- | --- | +| 2B | 64、128、256、512、1024、2048 | +| 4B | 64、128、256、512、1024、2560 | +| 9B | 64、128、256、512、1024、2048、4096 | + +这些值来自[官方模型配置与说明](https://github.com/Tencent/WeMM-Embedding#model-zoo),vLLM 会拒绝列表之外的维度。更换输出维度后需重建已有向量索引。超时、并发与图片大小限制保持现有默认值,按实际服务性能调整。 + +知识库还需开启图片向量索引才能直接检索图片内容。当前预设接入文本和图片;视频、任意图文交错输入及扫描 PDF 整页向量化不在当前业务链路的支持范围内。 + #### 字段参考 下表的「协议默认」是没有任何厂商、目录或覆盖时的取值;具体厂商的实际值以「实际调用方式」面板为准。 diff --git a/website-docs/06-development/06-wemm-transformers.md b/website-docs/06-development/06-wemm-transformers.md new file mode 100644 index 00000000..914f6d12 --- /dev/null +++ b/website-docs/06-development/06-wemm-transformers.md @@ -0,0 +1,205 @@ +# WeMM Transformers 部署 + +使用 [WeMM 官方 Transformers 推理代码](https://github.com/Tencent/WeMM-Embedding#transformers)加载模型,由仓库内的 `scripts/wemm-transformers/serve.py` 提供 HTTP Embedding 服务。服务可以与 WeKnora 部署在同一机器,也可以部署在独立 GPU 服务器上,通过 Base URL 接入。 + +CUDA、Apple MPS 和 CPU 是可选的计算设备,使用同一份脚本和请求协议。此 HTTP 适配层由 WeKnora 提供,支持文本和图片;模型本身支持的视频等能力尚未在此接口开放。官方 vLLM 部署方式见[模型配置](../03-features/06-models.md#wemm-embedding)。 + +## 安装依赖 + +以下命令在 WeKnora 仓库根目录执行,使用 Python 3.12。虚拟环境和模型文件放在仓库外: + +```bash +python3.12 -m venv "$HOME/.venvs/weknora-wemm" +source "$HOME/.venvs/weknora-wemm/bin/activate" +python -m pip install --upgrade pip +``` + +按设备选择一组 PyTorch 安装命令。以下使用已验证的 PyTorch 2.10.0 / torchvision 0.25.0,CUDA wheel 须与驱动兼容;其他 CUDA 版本可参考 [PyTorch 安装说明](https://pytorch.org/get-started/previous-versions/#v2100)。 + +```bash +# NVIDIA GPU:示例为 CUDA 12.8 wheel +python -m pip install torch==2.10.0 torchvision==0.25.0 \ + --index-url https://download.pytorch.org/whl/cu128 + +# 或 Apple Silicon / MPS +python -m pip install torch==2.10.0 torchvision==0.25.0 + +# 或 Linux / Windows CPU wheel +python -m pip install torch==2.10.0 torchvision==0.25.0 \ + --index-url https://download.pytorch.org/whl/cpu +``` + +安装其余依赖: + +```bash +python -m pip install -r scripts/wemm-transformers/requirements.txt +``` + +Transformers 固定为官方推荐的 `5.2.0`,`qwen-vl-utils` 固定为 `0.0.14`。不要随意替换 Transformers 版本,避免预处理行为变化。脚本会启用 `trust_remote_code=True` 来加载 WeMM 的 `embedding()` 实现,应使用官方权重或已审查的模型目录。 + +## 启动服务 + +以 CUDA 上的 2B 为例: + +```bash +python scripts/wemm-transformers/serve.py \ + --model tencent/WeMM-Embedding-2B \ + --revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \ + --device cuda \ + --host 127.0.0.1 \ + --port 18080 +``` + +首次启动会从 Hugging Face 下载模型到标准缓存,后续复用缓存。`--revision` 可固定模型代码与权重版本;示例 SHA 属于 2B,切换 4B、9B 时应选取对应仓库的版本,不能沿用此 SHA。未指定时使用该仓库默认版本。 + +已有模型目录时可以离线启动,并指定 WeKnora 调用时使用的模型名称: + +```bash +python scripts/wemm-transformers/serve.py \ + --model /srv/models/WeMM-Embedding-2B \ + --served-model-name tencent/WeMM-Embedding-2B \ + --local-files-only \ + --device cuda +``` + +也可以提前下载并固定版本,例如: + +```bash +hf download tencent/WeMM-Embedding-2B \ + --revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \ + --local-dir /srv/models/WeMM-Embedding-2B +``` + +其他设备只需更换参数: + +```bash +# Apple Silicon,允许不支持的 MPS 算子回退到 CPU +python scripts/wemm-transformers/serve.py \ + --model tencent/WeMM-Embedding-2B --device mps --mps-fallback + +# CPU,默认使用 float32 +python scripts/wemm-transformers/serve.py \ + --model tencent/WeMM-Embedding-2B --device cpu +``` + +4B、9B 将 `--model` 换成对应 ID 或路径。服务从模型配置读取原生维度及 Matryoshka 可选维度,不会将其他规格强制按 2048 维处理: + +| 模型 ID | 原生维度 | +| --- | --- | +| `tencent/WeMM-Embedding-2B` | 2048 | +| `tencent/WeMM-Embedding-4B` | 2560 | +| `tencent/WeMM-Embedding-9B` | 4096 | + +服务以前台方式运行,按 Ctrl+C 停止。常驻部署时由 systemd、容器或其他进程管理器托管同一命令。一个进程加载一份完整模型到一个设备,不提供多 GPU 切分;选择规格前应预留权重、视觉处理和推理激活所需的显存/内存。 + +### 独立服务器与鉴权 + +跨机器部署时使用 `--host 0.0.0.0`,并在服务进程环境中设置 `WEMM_API_KEY`。`/v1/models` 与 `/v1/embeddings` 会验证 `Authorization: Bearer `;WeKnora 的 API Key 填写相同值。服务不直接终止 TLS,跨不可信网络时通过 HTTPS 反向代理访问。`/health` 仅提供运行状态和非敏感模型元数据,不要求 API Key。 + +默认监听回环地址、未设置 API Key。WeKnora 若运行在 Docker 中,`127.0.0.1` 指向 WeKnora 容器自身,应填写容器能够访问的推理服务地址,例如同一容器网络的服务名,或已配置的宿主机地址。私网地址还需符合 WeKnora 的 SSRF 访问策略,配置方法见[安全说明](../01-getting-started/04-configuration.md)。 + +## WeKnora 接入 + +在「设置 → 模型管理」添加: + +| 字段 | 填写 | +| --- | --- | +| 类型 | Embedding | +| 来源 / 服务商 | API / 自定义(OpenAI 兼容接口) | +| 模型名称 | `/v1/models` 返回的 ID,推荐保留 `tencent/WeMM-Embedding-2B` 等官方名称 | +| Base URL | `http://推理服务地址:18080/v1`,使用 HTTPS 代理时填写对应地址 | +| API Key | 设置了 `WEMM_API_KEY` 时填写相同值 | +| 图片输入 | 官方名称由预设识别;自定义别名需手动开启 | +| 自定义输出维度 | 初次接入关闭,让连接检测识别原生维度;需要降维时再开启 | +| 后台并发上限 | 建议从 1 开始 | + +现有 WeMM 预设的 `messages` 格式同时适用于此服务。推荐在「高级 → 协议兼容覆盖」设置较小批量和较长超时: + +```json +{ + "api": "openai-embeddings", + "image_format": "messages", + "text_as_messages": true, + "batch_messages": true, + "accepts_truncate_prompt_tokens": false, + "max_batch_size": 4, + "max_image_batch_size": 2, + "request_timeout_seconds": 180 +} +``` + +完整配置也适用于尚未包含 WeMM 预设的旧版本或自定义模型别名;此时应手动打开图片输入。点击「测试连接」,确认实际维度后保存。知识库还需开启图片向量索引,配置方式见[模型配置](../03-features/06-models.md)。切换模型或输出维度需要重建已有索引。 + +## 请求格式与推理流程 + +接口为 `POST /v1/embeddings`。纯文本可以使用 `input`: + +```json +{ + "model": "tencent/WeMM-Embedding-2B", + "input": ["第一段文本", "第二段文本"], + "encoding_format": "float" +} +``` + +文本和图片也可使用 `messages`。以下二维数组包含两个独立输入,分别返回一个文本向量和一个图片向量: + +```json +{ + "model": "tencent/WeMM-Embedding-2B", + "dimensions": 1024, + "messages": [ + [{"role": "user", "content": [{"type": "text", "text": "白色背景上的红色方块"}]}], + [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,图片Base64"}}]}] + ] +} +``` + +`input` 和 `messages` 只能选一个。单条 conversation 可以使用一维消息数组;文字与图片放在同一 conversation 中会得到一个联合向量。返回值采用 OpenAI Embedding 的 `data[].embedding`、`data[].index` 和 `usage` 结构,编码支持 `float` 与 float32 小端字节的 `base64`。 + +服务会将文本包装成消息,把内联图片解码成 PIL 图片,使用官方 processor 的聊天模板,再转换为张量。Tokenizer 自动追加 ``,调用方无需手动添加。推理调用 `model.embedding(..., use_cache=False)`,随后按请求维度截取并做 L2 归一化。这里使用 Transformers 自带的模型 tokenizer/processor;vLLM 的 `embedding_chat_template.jinja` 不作为本脚本的额外模板传入。 + +HTTP 支持批量输入,但模型逐条执行,并串行处理推理请求,以隔离模型内部的可变位置状态。`batch_messages: true` 表示合并 HTTP 请求,不代表 GPU 张量批处理,也不提供 vLLM 的连续批处理调度。 + +## 参数与限制 + +| 参数 | 默认值 | 说明 | +| --- | --- | --- | +| `--device` | `auto` | 自动按 CUDA → MPS → CPU 选择;可显式指定 `cuda:N`、`mps`、`cpu`,指定不可用设备会报错 | +| `--dtype` | `auto` | CUDA 支持 BF16 时用 BF16,否则 FP16;MPS 用 BF16;CPU 用 FP32。可显式指定 `bfloat16`、`float16`、`float32` | +| `--max-tokens` | 8192 | 每个输入经过模板和图片处理后的总 token 上限,超限拒绝,不静默截断 | +| `--max-pixels` | 1048576 | 每张图片缩放后的像素上限,至少 65536;较小上限会影响高分辨率文档效果 | +| `--max-batch-size` | 32 | 每个 HTTP 请求的独立输入数上限;WeKnora 的批量设置不应超过它 | +| `--cpu-threads` | 8 | PyTorch CPU 线程数 | +| `--mps-fallback` | 关闭 | 允许 PyTorch 将不支持的 MPS 算子放到 CPU 执行 | +| `--local-files-only` | 关闭 | 仅使用本机文件或已有缓存,不下载权重 | +| `--host` / `--port` | `127.0.0.1` / `18080` | 监听地址与端口 | + +每个请求体最多 32 MiB,包含分块传输;每张图片的 data URI 最多 16 MiB,解码前原图最多 4000 万像素,每个 conversation 最多 5 张图片。图片只接受内联 Base64,不抓取远程 URL 或文件路径。`image_url.detail` 仅为格式兼容,实际分辨率由 `--max-pixels` 控制。所有上限都是此部署脚本的资源设置,不代表 WeMM 模型本身的上限。 + +## 验证与排障 + +运行无需下载权重的接口测试: + +```bash +python -m unittest discover -s scripts/wemm-transformers -v +``` + +对运行中的服务做实际图文推理测试: + +```bash +python scripts/wemm-transformers/smoke_test.py \ + --base-url http://127.0.0.1:18080 +``` + +设置了鉴权时,测试进程同样读取 `WEMM_API_KEY`。测试覆盖批量图文、输出维度与归一化、图文匹配、图片后文本状态隔离及降维;输出耗时和结果到标准输出,不修改知识库。 + +当前在 Apple M4 Pro 上完成了 **WeMM 2B / MPS / BF16** 和 **WeMM 2B / CPU / FP32** 的完整图文推理测试。2B、4B、9B 的接口维度处理及 CUDA 精度选择有自动化覆盖;4B/9B 的完整权重推理和 CUDA 部署仍需在目标设备运行上述验证脚本。不要将接口测试视为这些设备和规格的性能验证。 + +常见问题: + +- **CUDA/MPS 不可用**:检查 PyTorch wheel、驱动与设备,或显式改为 `--device cpu`;脚本不会将显式设备要求静默降级。 +- **MPS 算子不支持**:尝试 `--mps-fallback`;BF16 不可用时可使用 `--dtype float32`,同时检查内存余量。 +- **显存不足或超时**:减小模型规格、图片像素、输入长度和 WeKnora 批量。该服务为单 worker,增加 HTTP 并发不会自动增加推理吞吐。 +- **模型加载缓慢**:首次需要下载权重;先预下载并固定版本,离线启动时检查目录中同时包含权重、tokenizer、processor 和官方 Python 模型代码。 +- **模型名不匹配**:检查 `/v1/models`;使用自定义目录时建议显式设置 `--served-model-name`。