diff --git a/Makefile b/Makefile index 816ea501..67f74ac7 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ test: .PHONY: model-catalog-check model-catalog-check: go test ./internal/models/parity/ ./internal/models/vendors/ ./internal/models/catalog/ ./internal/models/api/... \ - ./internal/models/rerank/ ./internal/models/embedding/ + ./internal/models/rerank/ ./internal/models/embedding/ ./internal/models/asr/ # Vendor catalog: report where our model metadata differs from models.dev. # Development aid only — nothing is fetched at runtime and nothing is written diff --git a/go.mod b/go.mod index f41067f7..20ae0700 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( github.com/redis/go-redis/v9 v9.14.1 github.com/robfig/cron/v3 v3.0.1 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 - github.com/sashabaranov/go-openai v1.41.2 github.com/sirupsen/logrus v1.9.4 github.com/slack-go/slack v0.23.1 github.com/spf13/viper v1.21.0 diff --git a/go.sum b/go.sum index e32a11df..528734cf 100644 --- a/go.sum +++ b/go.sum @@ -2516,8 +2516,6 @@ github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM= -github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 10ed2f4f..6eac5651 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -18,6 +18,7 @@ import ( "github.com/Tencent/WeKnora/internal/infrastructure/chunker" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/searchutil" @@ -3699,7 +3700,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return nil } - transcriptionResult, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName) + // The knowledge base's language hint; the vendor declares where it + // goes, or that it takes none. + asrCtx := asr.WithLanguage(ctx, eff.ASRConfig.Language) + transcriptionResult, err := asrModel.Transcribe(asrCtx, convertResult.AudioData, knowledge.FileName) if err != nil { logger.Errorf(ctx, "[ASR] Transcription failed: %v", err) if isLastRetry { diff --git a/internal/handler/model.go b/internal/handler/model.go index c2f2a330..2a73e723 100644 --- a/internal/handler/model.go +++ b/internal/handler/model.go @@ -759,8 +759,7 @@ func (h *ModelHandler) DeleteModel(c *gin.Context) { // validateCatalogParameters rejects protocol / compat overrides the catalog // cannot interpret (unknown extra_config.api, unknown compat keys, bad // reasoning levels) so a typo fails at save time instead of at the first -// call. Chat, VLM, embedding and rerank rows are checked; see -// catalog.ValidateRow. +// call. Every model type is checked; see catalog.ValidateRow. func validateCatalogParameters(name string, modelType types.ModelType, params *types.ModelParameters) error { return catalog.ValidateRow(name, modelType, params) } diff --git a/internal/models/api/endpoint.go b/internal/models/api/endpoint.go index e9f7ddd9..5075753a 100644 --- a/internal/models/api/endpoint.go +++ b/internal/models/api/endpoint.go @@ -100,24 +100,33 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b if err != nil { return nil, nil, fmt.Errorf("marshal request: %w", err) } - if err := secutils.ValidateURLForSSRF(url); err != nil { - return nil, nil, fmt.Errorf("endpoint SSRF check failed: %w", err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) + req, err := e.newPost(ctx, url, data, "application/json") if err != nil { - return nil, nil, fmt.Errorf("create request: %w", err) + return nil, nil, err } - req.Header.Set("Content-Type", "application/json") if stream { req.Header.Set("Accept", "text/event-stream") } + return req, data, nil +} + +// newPost prepares an authenticated POST of an already-encoded body. +func (e Endpoint) newPost(ctx context.Context, url string, data []byte, contentType string) (*http.Request, error) { + if err := secutils.ValidateURLForSSRF(url); err != nil { + return nil, fmt.Errorf("endpoint SSRF check failed: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", contentType) if e.Auth != nil { e.Auth(req, data) } // User headers are applied last but the helper skips reserved names so // they cannot clobber auth or content negotiation. secutils.ApplyCustomHeaders(req, e.Headers) - return req, data, nil + return req, nil } // Do sends the request and returns the response. Non-2xx responses are @@ -125,7 +134,7 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b func (e Endpoint) Do(req *http.Request) (*http.Response, error) { resp, err := e.httpClient().Do(req) if err != nil { - return nil, fmt.Errorf("send request: %w", err) + return nil, &TransportError{Op: "send request", Err: err} } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) @@ -151,6 +160,11 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error // prompt is what an operator debugs, while a rerank body is a query plus // every candidate chunk. The rerank layer logs a truncated line at Debug // instead, so this would have duplicated it at Info. + return e.roundTrip(req, out) +} + +// roundTrip sends a prepared request and decodes a 2xx reply into out. +func (e Endpoint) roundTrip(req *http.Request, out any) error { resp, err := e.Do(req) if err != nil { return err @@ -158,7 +172,9 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error defer func() { _ = resp.Body.Close() }() raw, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("read response: %w", err) + // The connection broke before the reply arrived whole: nothing was + // received, so this is the network failing, not the vendor answering. + return &TransportError{Op: "read response", Err: err} } if out == nil { return nil @@ -169,6 +185,18 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error return nil } +// TransportError is a request that never got a whole answer: DNS, connect, +// TLS, a reset or a timeout, while sending or while reading the reply. It is +// the only failure worth sending again. +type TransportError struct { + // Op is the phase that failed: "send request" or "read response". + Op string + Err error +} + +func (e *TransportError) Error() string { return e.Op + ": " + e.Err.Error() } +func (e *TransportError) Unwrap() error { return e.Err } + // HTTPError is a non-2xx vendor reply. type HTTPError struct { StatusCode int diff --git a/internal/models/api/multipart.go b/internal/models/api/multipart.go new file mode 100644 index 00000000..be47a189 --- /dev/null +++ b/internal/models/api/multipart.go @@ -0,0 +1,76 @@ +package api + +import ( + "bytes" + "context" + "fmt" + "mime/multipart" + "net/textproto" + "path/filepath" + "strings" +) + +// FormField is one text part of a multipart form. Order is kept. +type FormField struct { + Name string + Value string +} + +// FormFile is the file part of a multipart form. +type FormFile struct { + Field string + FileName string + Data []byte +} + +// EncodeMultipart writes fields and one file into a multipart/form-data body +// and returns it with its Content-Type. It is encoded once: those bytes are +// what a signing vendor hashes and what every retry sends again. +// +// The file part carries a filename and no Content-Type of its own. Servers +// identify the format from the extension then — OpenAI asks for "an +// extension-bearing filename", and vox-box (GPUStack's audio backend) falls +// back to guessing from it — whereas a guessed type that a server does not +// list is rejected outright. +func EncodeMultipart(fields []FormField, file FormFile) ([]byte, string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, + file.Field, filepath.Base(file.FileName))) + part, err := w.CreatePart(header) + if err != nil { + return nil, "", fmt.Errorf("encode form file: %w", err) + } + if _, err := part.Write(file.Data); err != nil { + return nil, "", fmt.Errorf("encode form file: %w", err) + } + for _, f := range fields { + if err := w.WriteField(f.Name, f.Value); err != nil { + return nil, "", fmt.Errorf("encode form field %s: %w", f.Name, err) + } + } + if err := w.Close(); err != nil { + return nil, "", fmt.Errorf("encode form: %w", err) + } + return buf.Bytes(), w.FormDataContentType(), nil +} + +// PostMultipartWithRetry sends an encoded multipart body, retrying transport +// failures only, and decodes the JSON reply into out. It has the guarantees +// of PostJSONWithRetry. +func (e Endpoint) PostMultipartWithRetry( + ctx context.Context, url string, body []byte, contentType string, + out any, policy RetryPolicy, label string, +) error { + if !strings.HasPrefix(contentType, "multipart/") { + return fmt.Errorf("%s: not a multipart content type: %q", label, contentType) + } + return withRetry(ctx, policy, label, func() error { + req, err := e.newPost(ctx, url, body, contentType) + if err != nil { + return err + } + return e.roundTrip(req, out) + }) +} diff --git a/internal/models/api/openaichataudio/client.go b/internal/models/api/openaichataudio/client.go new file mode 100644 index 00000000..1e71ddbd --- /dev/null +++ b/internal/models/api/openaichataudio/client.go @@ -0,0 +1,152 @@ +// Package openaichataudio implements speech recognition served through the +// chat completions endpoint: one user message whose only content part is +// {"type": "input_audio", "input_audio": {"data": "data:;base64,..."}}, +// answered by an assistant message whose content is the transcript. +// +// Only dedicated recognition models are addressed this way — Alibaba's +// qwen3-asr-flash and Xiaomi's mimo-v2.5-asr — so no instruction is sent: the +// model transcribes whatever it is given. A general audio-capable chat model +// would need a prompt and would answer about the audio rather than transcribe +// it, which is why this is its own protocol and not a chat request. +// +// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference +// https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition +package openaichataudio + +import ( + "context" + "encoding/base64" + "fmt" + "path/filepath" + "strings" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" +) + +// Config is everything the client needs, already resolved by the catalog. +type Config struct { + Endpoint api.Endpoint + Settings catalog.TranscriptionsSettings + Retry api.RetryPolicy +} + +// Client talks chat-served speech recognition to one endpoint. +type Client struct { + cfg Config +} + +// New builds a client. +func New(cfg Config) *Client { return &Client{cfg: cfg} } + +const defaultPath = "/chat/completions" + +func (c *Client) url() string { + if c.cfg.Endpoint.URL != "" { + return c.cfg.Endpoint.Resolve("") + } + path := defaultPath + if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) { + path = "" + } + return c.cfg.Endpoint.Resolve(path) +} + +// audioTypes maps the file extension onto the media type of the data URI. +// The two references show audio/wav and audio/mpeg; the rest are the +// registered types for the containers Alibaba lists. A fixed table rather +// than mime.TypeByExtension, which reads the host's tables and would make the +// request depend on the machine it runs on. +var audioTypes = map[string]string{ + ".wav": "audio/wav", + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".mp4": "audio/mp4", + ".aac": "audio/aac", + ".flac": "audio/flac", + ".ogg": "audio/ogg", + ".opus": "audio/opus", + ".webm": "audio/webm", + ".amr": "audio/amr", +} + +// DataURI is the input_audio payload for one file. Both references show an +// audio media type in the URI, so a file whose extension names no audio +// container is refused rather than labelled application/octet-stream. +func DataURI(audio []byte, fileName string) (string, error) { + mediaType, ok := audioTypes[strings.ToLower(filepath.Ext(fileName))] + if !ok { + return "", fmt.Errorf("cannot tell the audio format of %q from its extension", fileName) + } + return "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(audio), nil +} + +// BuildRequestBody is the golden-test entry point. +func (c *Client) BuildRequestBody(req api.TranscriptionRequest) (map[string]any, error) { + uri, err := DataURI(req.Audio, req.FileName) + if err != nil { + return nil, err + } + // The ceiling is on the whole string as sent — "encoded string size must + // not exceed 10 MB" — so the data: prefix counts too. + if limit := c.cfg.Settings.MaxEncodedBytes; limit > 0 && len(uri) > limit { + return nil, fmt.Errorf( + "the audio is %.1f MB and %.1f MB once base64-encoded; this model accepts at most %.1f MB encoded", + float64(len(req.Audio))/(1<<20), float64(len(uri))/(1<<20), float64(limit)/(1<<20)) + } + body := map[string]any{ + "model": c.cfg.Endpoint.Model, + "messages": []any{map[string]any{ + "role": "user", + "content": []any{map[string]any{ + "type": "input_audio", + "input_audio": map[string]any{"data": uri}, + }}, + }}, + } + if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageASROptions { + body["asr_options"] = map[string]any{"language": req.Language} + } + return body, nil +} + +type response struct { + Choices []struct { + Message struct { + // Content is a pointer for the same reason as the multipart + // protocol's text: an absent transcript is an error, an empty one + // is silence. + Content *string `json:"content"` + } `json:"message"` + } `json:"choices"` + // Usage.Seconds is the audio length, which Alibaba reports. + Usage struct { + Seconds float64 `json:"seconds"` + } `json:"usage"` + Error *struct { + Message string `json:"message"` + } `json:"error"` +} + +// Transcribe sends one audio file. +func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) { + body, err := c.BuildRequestBody(req) + if err != nil { + return nil, err + } + var decoded response + err = c.cfg.Endpoint.PostJSONWithRetry(ctx, c.url(), body, &decoded, c.cfg.Retry, "transcription") + if err != nil { + return nil, err + } + if decoded.Error != nil && decoded.Error.Message != "" { + return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message) + } + if len(decoded.Choices) == 0 || decoded.Choices[0].Message.Content == nil { + return nil, fmt.Errorf("transcription reply carries no message content") + } + return &api.Transcription{ + Text: strings.TrimSpace(*decoded.Choices[0].Message.Content), + Duration: decoded.Usage.Seconds, + }, nil +} diff --git a/internal/models/api/openaichataudio/golden_test.go b/internal/models/api/openaichataudio/golden_test.go new file mode 100644 index 00000000..8515a02f --- /dev/null +++ b/internal/models/api/openaichataudio/golden_test.go @@ -0,0 +1,162 @@ +package openaichataudio + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// qwenASRResponse is the non-streaming response example from Alibaba's +// Qwen-ASR reference, copied from the page: +// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference +const qwenASRResponse = `{ + "choices": [{ + "finish_reason": "stop", + "index": 0, + "message": { + "annotations": [{"emotion": "neutral", "language": "zh", "type": "audio_info"}], + "content": "欢迎使用阿里云。", + "role": "assistant" + } + }], + "created": 1767683986, + "id": "chatcmpl-487abe5f-d4f2-9363-a877-xxxxxxx", + "model": "qwen3-asr-flash", + "object": "chat.completion", + "usage": {"completion_tokens": 12, "prompt_tokens": 42, "total_tokens": 54} +}` + +// withSeconds is a reply carrying the usage.seconds that the reference's +// schema documents ("音频时长(秒)") but its example omits. +const withSeconds = `{"choices":[{"message":{"role":"assistant","content":"x"}}],"usage":{"seconds":3}}` + +func newClient(url, model string) *Client { + return newClientWith(url, model, catalog.TranscriptionsSettings{}) +} + +func newClientWith(url, model string, settings catalog.TranscriptionsSettings) *Client { + return New(Config{ + Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")}, + Settings: settings, + }) +} + +// The request is the one both references show: a single user message whose +// only part is the audio as a data URI, no instruction, no stream flag. +func TestRequestBodyMatchesTheReferences(t *testing.T) { + body, err := newClient("https://example.invalid/v1", "qwen3-asr-flash"). + BuildRequestBody(api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "clip.MP3"}) + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "model": "qwen3-asr-flash", + "messages": []any{map[string]any{ + "role": "user", + "content": []any{map[string]any{ + "type": "input_audio", + "input_audio": map[string]any{"data": "data:audio/mpeg;base64,SUQz"}, + }}, + }}, + }, body) +} + +func TestDataURIMediaTypeFollowsTheExtension(t *testing.T) { + uri, err := DataURI([]byte("RIFF"), "a.wav") + require.NoError(t, err) + assert.Equal(t, "data:audio/wav;base64,UklGRg==", uri) + + // No audio type can be named for it, and the references show one. + _, err = DataURI([]byte("x"), "noext") + assert.Error(t, err) +} + +// Both references cap the string as sent, so the data: prefix counts. A file +// sized to exactly the encoded budget without the prefix goes over it. +func TestEncodedCeilingCountsTheWholeDataURI(t *testing.T) { + const limit = 1024 + c := newClientWith("https://example.invalid", "m", catalog.TranscriptionsSettings{MaxEncodedBytes: limit}) + audio := make([]byte, limit*3/4) // base64 of this is exactly limit bytes + _, err := c.BuildRequestBody(api.TranscriptionRequest{Audio: audio, FileName: "a.wav"}) + require.Error(t, err) + + prefix := len("data:audio/wav;base64,") + fits := make([]byte, (limit-prefix)/4*3) + _, err = c.BuildRequestBody(api.TranscriptionRequest{Audio: fits, FileName: "a.wav"}) + require.NoError(t, err) +} + +func TestLanguageGoesInASROptionsWhereDeclared(t *testing.T) { + req := api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav", Language: "zh"} + + body, err := newClient("https://example.invalid", "m").BuildRequestBody(req) + require.NoError(t, err) + assert.NotContains(t, body, "asr_options", "undeclared, so not sent") + + body, err = newClientWith("https://example.invalid", "m", + catalog.TranscriptionsSettings{LanguageParam: catalog.LanguageASROptions}).BuildRequestBody(req) + require.NoError(t, err) + assert.Equal(t, map[string]any{"language": "zh"}, body["asr_options"]) +} + +func TestDecodesTheDocumentedResponse(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + var path string + var sent map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&sent) + _, _ = w.Write([]byte(qwenASRResponse)) + })) + defer server.Close() + + out, err := newClient(server.URL+"/compatible-mode/v1", "qwen3-asr-flash"). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "a.wav"}) + require.NoError(t, err) + assert.Equal(t, "/compatible-mode/v1/chat/completions", path) + assert.Equal(t, &api.Transcription{Text: "欢迎使用阿里云。"}, out) +} + +// A reply with no assistant content is not silence. +func TestReplyWithoutContentIsAnError(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer server.Close() + + _, err := newClient(server.URL, "m"). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + assert.Error(t, err) +} + +func TestDurationFromUsageSeconds(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(withSeconds)) + })) + defer server.Close() + + out, err := newClient(server.URL, "m"). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.NoError(t, err) + assert.Equal(t, float64(3), out.Duration) +} + +func TestEmptyContentIsSilence(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":""}}]}`)) + })) + defer server.Close() + + out, err := newClient(server.URL, "m"). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.NoError(t, err) + assert.Equal(t, "", out.Text) +} diff --git a/internal/models/api/openaitranscriptions/client.go b/internal/models/api/openaitranscriptions/client.go new file mode 100644 index 00000000..b09762ca --- /dev/null +++ b/internal/models/api/openaitranscriptions/client.go @@ -0,0 +1,131 @@ +// Package openaitranscriptions implements the speech-to-text shape OpenAI +// defined and the compatible servers copied: a multipart POST to +// {base}/audio/transcriptions carrying file and model, answering {text}, or +// {text, segments} when response_format is verbose_json. +// +// response_format is the one optional field that matters, and it differs by +// model: gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json, +// whisper-1 also serves verbose_json, SiliconFlow documents no such field at +// all. json is the default everywhere it is documented, so nothing is sent +// unless catalog.TranscriptionsSettings names a format. +// +// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create +package openaitranscriptions + +import ( + "context" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" +) + +// Config is everything the client needs, already resolved by the catalog. +type Config struct { + Endpoint api.Endpoint + Settings catalog.TranscriptionsSettings + Retry api.RetryPolicy +} + +// Client talks the OpenAI transcription shape to one endpoint. +type Client struct { + cfg Config +} + +// New builds a client. +func New(cfg Config) *Client { return &Client{cfg: cfg} } + +func (c *Client) url() string { + if c.cfg.Endpoint.URL != "" { + return c.cfg.Endpoint.Resolve("") + } + path := c.cfg.Settings.Path + if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) { + path = "" + } + return c.cfg.Endpoint.Resolve(path) +} + +// FormFields is the golden-test entry point: the text parts of the form, in +// the order they are written. +func (c *Client) FormFields(language string) []api.FormField { + fields := []api.FormField{{Name: "model", Value: c.cfg.Endpoint.Model}} + if format := c.cfg.Settings.ResponseFormat; format != "" { + fields = append(fields, api.FormField{Name: "response_format", Value: format}) + } + if language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageForm { + fields = append(fields, api.FormField{Name: "language", Value: language}) + } + return fields +} + +type response struct { + // Text is a pointer because its absence is an error, not silence. A + // transcript of silent audio is a present, empty string; a reply without + // the field is some other document. vox-box (GPUStack's audio backend) + // returns its HTTPException from the route instead of raising it, so a + // failed transcription can arrive as a 2xx whose body is the exception. + Text *string `json:"text"` + Segments []struct { + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + } `json:"segments"` + // Duration is top-level in OpenAI's verbose_json and in MiniMax's + // default json; usage.seconds is where OpenAI, OpenRouter and Requesty + // report it for models billed by duration. + Duration float64 `json:"duration"` + Usage struct { + Seconds float64 `json:"seconds"` + } `json:"usage"` + Detail any `json:"detail"` + Error *struct { + Message string `json:"message"` + } `json:"error"` +} + +// Transcribe sends one audio file. +func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) { + body, contentType, err := api.EncodeMultipart(c.FormFields(req.Language), api.FormFile{ + Field: "file", FileName: req.FileName, Data: req.Audio, + }) + if err != nil { + return nil, err + } + endpoint := c.cfg.Endpoint + if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageHeader { + headers := make(map[string]string, len(endpoint.Headers)+1) + for k, v := range endpoint.Headers { + headers[k] = v + } + headers["language"] = req.Language + endpoint.Headers = headers + } + var decoded response + err = endpoint.PostMultipartWithRetry( + ctx, c.url(), body, contentType, &decoded, c.cfg.Retry, "transcription", + ) + if err != nil { + return nil, err + } + if decoded.Error != nil && decoded.Error.Message != "" { + return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message) + } + if decoded.Text == nil { + if decoded.Detail != nil { + return nil, fmt.Errorf("transcription reply carries no text: %v", decoded.Detail) + } + return nil, fmt.Errorf("transcription reply carries no text") + } + out := &api.Transcription{Text: strings.TrimSpace(*decoded.Text), Duration: decoded.Duration} + if out.Duration == 0 { + out.Duration = decoded.Usage.Seconds + } + for _, s := range decoded.Segments { + out.Segments = append(out.Segments, api.TranscriptionSegment{ + Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text), + }) + } + return out, nil +} diff --git a/internal/models/api/openaitranscriptions/golden_test.go b/internal/models/api/openaitranscriptions/golden_test.go new file mode 100644 index 00000000..2e2252b7 --- /dev/null +++ b/internal/models/api/openaitranscriptions/golden_test.go @@ -0,0 +1,219 @@ +package openaitranscriptions + +import ( + "context" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// jsonResponse is the json example from the reference, copied from the page: +// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create +const jsonResponse = `{ + "text": "text", + "languages": [{"code": "code"}], + "logprobs": [{"token": "token", "bytes": [0], "logprob": 0}], + "usage": { + "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "type": "tokens", + "input_token_details": {"audio_tokens": 0, "text_tokens": 0} + } +}` + +// verboseResponse is the "Segment timestamps" example from the same page, +// cut to its first segment where the page elides the rest with "...". +const verboseResponse = `{ + "task": "transcribe", + "language": "english", + "duration": 8.470000267028809, + "text": "The beach was a popular spot on a hot summer day. ` + + `People were swimming in the ocean, building sandcastles, and playing beach volleyball.", + "segments": [ + { + "id": 0, + "seek": 0, + "start": 0.0, + "end": 3.319999933242798, + "text": " The beach was a popular spot on a hot summer day.", + "tokens": [50364, 440, 7534, 390, 257, 3743, 4008, 322, 257, 2368, 4266, 786, 13, 50530], + "temperature": 0.0, + "avg_logprob": -0.2860786020755768, + "compression_ratio": 1.2363636493682861, + "no_speech_prob": 0.00985979475080967 + } + ], + "usage": {"type": "duration", "seconds": 9} +}` + +type captured struct { + path string + auth string + language string + fields map[string]string + fileName string + fileType string + file []byte +} + +func serve(t *testing.T, status int, reply string) (*httptest.Server, *captured) { + t.Helper() + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + got := &captured{fields: map[string]string{}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got.path = r.URL.Path + got.auth = r.Header.Get("Authorization") + got.language = r.Header.Get("language") + _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + require.NoError(t, err) + reader := multipart.NewReader(r.Body, params["boundary"]) + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err) + data, _ := io.ReadAll(part) + if part.FormName() == "file" { + got.fileName, got.fileType, got.file = part.FileName(), part.Header.Get("Content-Type"), data + continue + } + got.fields[part.FormName()] = string(data) + } + w.WriteHeader(status) + _, _ = w.Write([]byte(reply)) + })) + t.Cleanup(server.Close) + return server, got +} + +func newClient(url, model string, settings catalog.TranscriptionsSettings) *Client { + if settings.Path == "" { + settings.Path = catalog.DefaultTranscriptions().Path + } + return New(Config{ + Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")}, + Settings: settings, + }) +} + +// The baseline form is file and model. gpt-4o-transcribe accepts only json, +// which is the default, so no response_format is sent. +func TestBaselineFormIsFileAndModel(t *testing.T) { + server, got := serve(t, http.StatusOK, jsonResponse) + out, err := newClient(server.URL+"/v1", "gpt-4o-transcribe", catalog.TranscriptionsSettings{}). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "meeting.wav"}) + require.NoError(t, err) + + assert.Equal(t, "/v1/audio/transcriptions", got.path) + assert.Equal(t, "Bearer k", got.auth) + assert.Equal(t, map[string]string{"model": "gpt-4o-transcribe"}, got.fields) + assert.Equal(t, "meeting.wav", got.fileName, "the extension is how the server identifies the format") + assert.Empty(t, got.fileType, "a guessed type a server does not list is rejected outright") + assert.Equal(t, []byte("RIFF"), got.file) + assert.Equal(t, &api.Transcription{Text: "text"}, out) +} + +// The language hint goes where the vendor documents it: a form field on +// OpenAI's shape, a request header on MiniMax's, nowhere otherwise. +func TestLanguageGoesWhereTheVendorDeclares(t *testing.T) { + for _, tc := range []struct { + param string + wantField string + wantHeader string + }{ + {param: "", wantField: "", wantHeader: ""}, + {param: catalog.LanguageForm, wantField: "zh", wantHeader: ""}, + {param: catalog.LanguageHeader, wantField: "", wantHeader: "zh"}, + } { + t.Run("param="+tc.param, func(t *testing.T) { + server, got := serve(t, http.StatusOK, jsonResponse) + _, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{LanguageParam: tc.param}). + Transcribe(context.Background(), api.TranscriptionRequest{ + Audio: []byte("x"), FileName: "a.wav", Language: "zh", + }) + require.NoError(t, err) + assert.Equal(t, tc.wantField, got.fields["language"]) + assert.Equal(t, tc.wantHeader, got.language) + }) + } +} + +// The audio length is read from wherever the reply states it, so usage can +// be reported without asking for segments. +func TestDurationIsReadFromTheReply(t *testing.T) { + for name, reply := range map[string]string{ + // MiniMax's default json, from its reference. + "top-level duration": `{"text": "x", "duration": 26.325, "trace_id": "021785229015510a2c883cf675b9804d"}`, + // OpenRouter's reference example. + "usage.seconds": `{"text": "x", "usage": {"cost": 0.000508, "input_tokens": 83, "output_tokens": 30, ` + + `"seconds": 9.2, "total_tokens": 113}}`, + } { + t.Run(name, func(t *testing.T) { + server, _ := serve(t, http.StatusOK, reply) + out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.NoError(t, err) + assert.NotZero(t, out.Duration) + }) + } +} + +func TestVerboseJSONCarriesSegments(t *testing.T) { + server, got := serve(t, http.StatusOK, verboseResponse) + settings := catalog.TranscriptionsSettings{ResponseFormat: "verbose_json"} + out, err := newClient(server.URL+"/v1", "whisper-1", settings). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "a.mp3"}) + require.NoError(t, err) + + assert.Equal(t, "verbose_json", got.fields["response_format"]) + require.Len(t, out.Segments, 1) + assert.Equal(t, api.TranscriptionSegment{ + Start: 0, End: 3.319999933242798, Text: "The beach was a popular spot on a hot summer day.", + }, out.Segments[0]) +} + +// Silence is an empty transcript, and that is a valid answer. +func TestEmptyTextIsSilence(t *testing.T) { + server, _ := serve(t, http.StatusOK, `{"text": ""}`) + out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.NoError(t, err) + assert.Equal(t, "", out.Text) +} + +// A reply without text is not silence. vox-box returns its HTTPException +// from the route rather than raising it, so a failure can arrive as a 2xx +// body shaped like this; reading it as an empty transcript would store "no +// speech detected" for a file that was never transcribed. +func TestReplyWithoutTextIsAnError(t *testing.T) { + server, _ := serve(t, http.StatusOK, + `{"status_code": 500, "detail": "Failed to transcribe audio, boom", "headers": null}`) + _, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "Failed to transcribe audio") +} + +func TestSurfacesTheVendorErrorBody(t *testing.T) { + // A stand-in rejection in OpenAI's error envelope. + server, _ := serve(t, http.StatusBadRequest, `{"error":{"message":"stand-in rejection"}}`) + _, err := newClient(server.URL, "gpt-4o-transcribe", catalog.TranscriptionsSettings{}). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "stand-in rejection") +} + +func TestDoesNotDoubleThePath(t *testing.T) { + server, got := serve(t, http.StatusOK, jsonResponse) + _, err := newClient(server.URL+"/v1/audio/transcriptions", "m", catalog.TranscriptionsSettings{}). + Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"}) + require.NoError(t, err) + assert.Equal(t, "/v1/audio/transcriptions", got.path) +} diff --git a/internal/models/api/retry.go b/internal/models/api/retry.go index 2dd9a02e..64e61c75 100644 --- a/internal/models/api/retry.go +++ b/internal/models/api/retry.go @@ -49,12 +49,23 @@ func (p RetryPolicy) delay(attempt int) time.Duration { // Here the last error is captured in the enclosing scope on purpose and the // function cannot fall through without it. // -// Only transport failures are retried. A non-2xx reply is the vendor -// answering, and repeating a rejected request neither fixes it nor tells the -// operator anything new. +// Only transport failures are retried, including a reply cut off while it +// was being read. A non-2xx reply is the vendor answering, and repeating a +// rejected request neither fixes it nor tells the operator anything new; a +// reply that arrived whole but does not decode is the same answer every time, +// and sending it again would bill the request twice. func (e Endpoint) PostJSONWithRetry( ctx context.Context, url string, body, out any, policy RetryPolicy, label string, ) error { + return withRetry(ctx, policy, label, func() error { + return e.PostJSON(ctx, url, body, out) + }) +} + +// withRetry runs send until it succeeds, fails with anything other than a +// TransportError, or the budget runs out. The last error lives in the +// enclosing scope and the loop cannot fall through without it. +func withRetry(ctx context.Context, policy RetryPolicy, label string, send func() error) error { var lastErr error for attempt := 0; attempt <= policy.MaxRetries; attempt++ { if attempt > 0 { @@ -67,12 +78,12 @@ func (e Endpoint) PostJSONWithRetry( return ctx.Err() } } - err := e.PostJSON(ctx, url, body, out) + err := send() if err == nil { return nil } - var httpErr *HTTPError - if errors.As(err, &httpErr) { + var transportErr *TransportError + if !errors.As(err, &transportErr) { return err } lastErr = err diff --git a/internal/models/api/retry_test.go b/internal/models/api/retry_test.go index 4d7c8d14..b056a4da 100644 --- a/internal/models/api/retry_test.go +++ b/internal/models/api/retry_test.go @@ -78,6 +78,25 @@ func TestPostJSONWithRetryDoesNotRetryAVendorRejection(t *testing.T) { assert.Equal(t, 1, attempts, "a rejection must not be repeated") } +// A reply that arrived but does not decode is not a transport failure: the +// vendor did the work, and sending the request again bills it again. +func TestPostJSONWithRetryDoesNotRetryAnUndecodableReply(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts++ + _, _ = w.Write([]byte(`not json`)) + })) + defer server.Close() + + var out map[string]any + err := Endpoint{}.PostJSONWithRetry( + context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test", + ) + require.Error(t, err) + assert.Equal(t, 1, attempts) +} + func TestPostJSONWithRetryStopsWhenTheContextIsDone(t *testing.T) { t.Setenv("SSRF_WHITELIST", "127.0.0.1") server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) @@ -101,3 +120,54 @@ func TestRetryPolicyBackoffDoublesAndIsCapped(t *testing.T) { assert.Equal(t, 4*time.Second, p.delay(4), "capped") assert.Zero(t, RetryPolicy{MaxRetries: 2}.delay(1), "no base delay means no waiting") } + +// A reply cut off mid-body never arrived, so it is sent again. +func TestPostJSONWithRetryRetriesAReplyCutOffWhileReading(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts++ + if attempts == 1 { + // Promise more than is sent, then hang up. + w.Header().Set("Content-Length", "100") + _, _ = w.Write([]byte(`{"a":`)) + w.(http.Flusher).Flush() // the status and half a body are on the wire + hj, ok := w.(http.Hijacker) + require.True(t, ok) + conn, _, err := hj.Hijack() + require.NoError(t, err) + _ = conn.Close() + return + } + _, _ = w.Write([]byte(`{"a":1}`)) + })) + defer server.Close() + + var out map[string]any + err := Endpoint{}.PostJSONWithRetry( + context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test", + ) + require.NoError(t, err) + assert.Equal(t, 2, attempts) + assert.Equal(t, float64(1), out["a"]) +} + +// The phase that failed is named, so the log says whether the request or the +// reply was lost. +func TestReadFailureIsReportedAsSuch(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = w.Write([]byte(`{"a":`)) + w.(http.Flusher).Flush() + conn, _, err := w.(http.Hijacker).Hijack() + require.NoError(t, err) + _ = conn.Close() + })) + defer server.Close() + + err := Endpoint{}.PostJSON(context.Background(), server.URL, map[string]any{}, &map[string]any{}) + var transportErr *TransportError + require.ErrorAs(t, err, &transportErr) + assert.Equal(t, "read response", transportErr.Op) +} diff --git a/internal/models/api/transcriptions.go b/internal/models/api/transcriptions.go new file mode 100644 index 00000000..6ace55fa --- /dev/null +++ b/internal/models/api/transcriptions.go @@ -0,0 +1,56 @@ +package api + +import "context" + +// TranscriptionAPI names a speech-to-text wire protocol. Like RerankAPI and +// EmbeddingAPI it is its own type, so no other modality's value validates. +type TranscriptionAPI string + +// The transcription protocols WeKnora speaks. +const ( + // TranscriptionOpenAI is POST {base}/audio/transcriptions as a multipart + // form carrying file and model, answering {text} — or {text, segments} + // when verbose_json is asked for. + TranscriptionOpenAI TranscriptionAPI = "openai-transcriptions" + // TranscriptionChatAudio is a dedicated speech-recognition model behind + // POST {base}/chat/completions: the audio goes in as a base64 data URI in + // an input_audio content part and the transcript comes back as the + // assistant message. Alibaba's qwen3-asr-flash and Xiaomi's mimo-v2.5-asr + // take no instruction; the model only transcribes. + TranscriptionChatAudio TranscriptionAPI = "openai-chat-audio" +) + +// Known reports whether the value names a protocol this build implements. +func (a TranscriptionAPI) Known() bool { + return a == TranscriptionOpenAI || a == TranscriptionChatAudio +} + +// TranscriptionSegment is one timed stretch of a transcript. +type TranscriptionSegment struct { + Start float64 + End float64 + Text string +} + +// Transcription is what a protocol client returns. +type Transcription struct { + Text string + Segments []TranscriptionSegment + // Duration is the audio length in seconds when the reply states it — + // OpenAI-shaped duration or usage.seconds — and 0 otherwise. + Duration float64 +} + +// TranscriptionRequest is one file to transcribe. +type TranscriptionRequest struct { + Audio []byte + FileName string + // Language is the operator's hint, empty for auto-detection. Where it is + // sent, and whether at all, is the vendor's declaration. + Language string +} + +// Transcriber turns one audio file into text. +type Transcriber interface { + Transcribe(ctx context.Context, req TranscriptionRequest) (*Transcription, error) +} diff --git a/internal/models/asr/asr.go b/internal/models/asr/asr.go index c241d52e..913e5e22 100644 --- a/internal/models/asr/asr.go +++ b/internal/models/asr/asr.go @@ -2,6 +2,7 @@ package asr import ( "context" + "strings" "github.com/Tencent/WeKnora/internal/types" ) @@ -17,6 +18,27 @@ type Segment struct { type TranscriptionResult struct { Text string `json:"text"` Segments []Segment `json:"segments,omitempty"` + // Duration is the audio length in seconds when the vendor reports it, + // which several do without segments; 0 when unknown. + Duration float64 `json:"duration,omitempty"` +} + +type languageKey struct{} + +// WithLanguage attaches the operator's language hint (for example "zh") to +// a transcription. Empty and "auto" mean auto-detection, which is every +// vendor's default, and send nothing. +func WithLanguage(ctx context.Context, language string) context.Context { + return context.WithValue(ctx, languageKey{}, language) +} + +func languageFrom(ctx context.Context) string { + language, _ := ctx.Value(languageKey{}).(string) + language = strings.TrimSpace(language) + if strings.EqualFold(language, "auto") { + return "" + } + return language } // ASR defines the interface for Automatic Speech Recognition model operations. @@ -35,7 +57,10 @@ type Config struct { ModelName string APIKey string ModelID string - Language string // optional: specify language for transcription + // Provider is the vendor id stored on the row; empty detects it from + // BaseURL. + Provider string + ExtraConfig map[string]string // CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 CustomHeaders map[string]string } @@ -53,13 +78,14 @@ func ConfigFromModel(m *types.Model) *Config { BaseURL: m.Parameters.BaseURL, ModelName: m.Name, Source: m.Source, + Provider: m.Parameters.Provider, + ExtraConfig: m.Parameters.ExtraConfig, CustomHeaders: m.Parameters.CustomHeaders, } } // NewASR creates an ASR instance based on the provided configuration. -// All ASR vendors use the OpenAI-compatible /v1/audio/transcriptions API. func NewASR(config *Config) (ASR, error) { - a, err := NewOpenAIASR(config) + a, err := newASR(config) return wrapASRLangfuse(a, err) } diff --git a/internal/models/asr/config_from_model_test.go b/internal/models/asr/config_from_model_test.go index 7d2275ec..ecf3bf1d 100644 --- a/internal/models/asr/config_from_model_test.go +++ b/internal/models/asr/config_from_model_test.go @@ -14,6 +14,8 @@ func TestConfigFromModel(t *testing.T) { Parameters: types.ModelParameters{ BaseURL: "https://api.example.com/v1", APIKey: "sk", + Provider: "siliconflow", + ExtraConfig: map[string]string{"remote_model_name": "r"}, CustomHeaders: map[string]string{"X": "y"}, }, } @@ -24,6 +26,11 @@ func TestConfigFromModel(t *testing.T) { if cfg.BaseURL != "https://api.example.com/v1" || cfg.APIKey != "sk" { t.Errorf("connection fields mismatch: %+v", cfg) } + // The vendor decides the wire format; without it every row resolves to + // the generic runtime. + if cfg.Provider != "siliconflow" || cfg.ExtraConfig["remote_model_name"] != "r" { + t.Errorf("vendor fields not propagated: %+v", cfg) + } if cfg.CustomHeaders["X"] != "y" { t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders) } diff --git a/internal/models/asr/langfuse_wrapper.go b/internal/models/asr/langfuse_wrapper.go index 7564c059..d0bc7ecf 100644 --- a/internal/models/asr/langfuse_wrapper.go +++ b/internal/models/asr/langfuse_wrapper.go @@ -8,7 +8,8 @@ import ( // langfuseASR wraps an ASR implementation and reports each Transcribe call // as a Langfuse generation observation. Audio bytes are not uploaded — we -// record file name, audio size and duration (derived from segment end times) +// record file name, audio size and duration (as the vendor reports it, or +// from segment end times) // so usage can be billed by second/minute, which is how most ASR providers // price their services. type langfuseASR struct { @@ -44,8 +45,13 @@ func (l *langfuseASR) Transcribe(ctx context.Context, audioBytes []byte, fileNam if result != nil { output["text"] = result.Text output["segment_count"] = len(result.Segments) - if n := len(result.Segments); n > 0 { + // The vendor's own figure when it gives one; otherwise the end of + // the last segment, which only verbose replies carry. + duration = result.Duration + if n := len(result.Segments); duration == 0 && n > 0 { duration = result.Segments[n-1].End + } + if duration > 0 { output["duration_seconds"] = duration } } diff --git a/internal/models/asr/openai.go b/internal/models/asr/openai.go deleted file mode 100644 index 66dee48e..00000000 --- a/internal/models/asr/openai.go +++ /dev/null @@ -1,150 +0,0 @@ -package asr - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/Tencent/WeKnora/internal/logger" - secutils "github.com/Tencent/WeKnora/internal/utils" - openai "github.com/sashabaranov/go-openai" -) - -const ( - asrDefaultTimeout = 300 * time.Second // audio transcription can be slow -) - -// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API. -type OpenAIASR struct { - modelName string - modelID string - client *openai.Client - baseURL string - language string -} - -// NewOpenAIASR creates an OpenAI-compatible ASR instance. -func NewOpenAIASR(config *Config) (*OpenAIASR, error) { - if err := validateASRBaseURL(config.BaseURL); err != nil { - return nil, err - } - - apiCfg := openai.DefaultConfig(config.APIKey) - if config.BaseURL != "" { - apiCfg.BaseURL = config.BaseURL - } - httpClient := newASRHTTPClient(asrDefaultTimeout) - - // 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers) - if len(config.CustomHeaders) > 0 { - apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders) - } else { - apiCfg.HTTPClient = httpClient - } - - return &OpenAIASR{ - modelName: config.ModelName, - modelID: config.ModelID, - client: openai.NewClientWithConfig(apiCfg), - baseURL: config.BaseURL, - language: config.Language, - }, nil -} - -// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API. -func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error) { - if len(audioBytes) == 0 { - return nil, fmt.Errorf("audio bytes are empty") - } - - // Ensure fileName has an extension for proper MIME type detection - if fileName == "" { - fileName = "audio.mp3" - } - - logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s", - s.modelName, s.baseURL, len(audioBytes), fileName) - - req := openai.AudioRequest{ - Model: s.modelName, - FilePath: fileName, - Reader: bytes.NewReader(audioBytes), - Format: openai.AudioResponseFormatVerboseJSON, - } - - if s.language != "" { - req.Language = s.language - } - - resp, err := s.client.CreateTranscription(ctx, req) - if err != nil { - return nil, fmt.Errorf("ASR transcription request failed: %w", err) - } - - jsonData, err := json.Marshal(resp) - if err != nil { - return nil, fmt.Errorf("marshal transcription response: %w", err) - } - logger.Debugf(ctx, "[ASR] Transcription response: %s", string(jsonData)) - - text := strings.TrimSpace(resp.Text) - logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text)) - - var segments []Segment - for _, s := range resp.Segments { - segments = append(segments, Segment{ - Start: s.Start, - End: s.End, - Text: strings.TrimSpace(s.Text), - }) - } - - return &TranscriptionResult{ - Text: text, - Segments: segments, - }, nil -} - -func (s *OpenAIASR) GetModelName() string { return s.modelName } -func (s *OpenAIASR) GetModelID() string { return s.modelID } - -// DetectAudioFormat returns a file extension hint for the given audio bytes. -func DetectAudioFormat(data []byte, fileName string) string { - if fileName != "" { - ext := strings.ToLower(filepath.Ext(fileName)) - if ext != "" { - return ext - } - } - // Try to detect from magic bytes - if len(data) >= 4 { - // MP3: starts with ID3 or 0xFF 0xFB - if data[0] == 'I' && data[1] == 'D' && data[2] == '3' { - return ".mp3" - } - if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 { - return ".mp3" - } - // FLAC: starts with "fLaC" - if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' { - return ".flac" - } - // OGG: starts with "OggS" - if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' { - return ".ogg" - } - // WAV: starts with "RIFF" - if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' { - return ".wav" - } - } - // M4A: check for ftyp box - if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' { - return ".m4a" - } - return ".mp3" // default -} diff --git a/internal/models/asr/protocol.go b/internal/models/asr/protocol.go new file mode 100644 index 00000000..b16b56ae --- /dev/null +++ b/internal/models/asr/protocol.go @@ -0,0 +1,183 @@ +package asr + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/api/openaichataudio" + "github.com/Tencent/WeKnora/internal/models/api/openaitranscriptions" + "github.com/Tencent/WeKnora/internal/models/catalog" + // catalog.Resolve answers from the vendor catalog, which is empty until + // the vendor packages have run their init. + _ "github.com/Tencent/WeKnora/internal/models/vendors" + "github.com/Tencent/WeKnora/internal/types" +) + +// retryPolicy is the transport-error retry budget, which is none. The +// knowledge pipeline retries a failed transcription as a whole task, and a +// client-level retry on top would turn one hung 300-second upload into +// several. +var retryPolicy = func() api.RetryPolicy { return api.RetryPolicy{} } + +// newASR resolves the catalog and returns the protocol client for the +// configured model. It mirrors rerank.newReranker and +// embedding.newRemoteEmbedder: the vendor's facts decide the protocol, the +// URL and the credential, and this function knows no vendor names. +func newASR(config *Config) (ASR, error) { + if config == nil { + return nil, fmt.Errorf("asr config is nil") + } + if strings.TrimSpace(config.ModelName) == "" { + return nil, fmt.Errorf("model name is required") + } + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: config.Provider, + Model: config.ModelName, + BaseURL: config.BaseURL, + ModelType: types.ModelTypeASR, + Extra: config.ExtraConfig, + }) + if err != nil { + return nil, err + } + // A vendor that does not declare speech recognition has not been checked + // for it, and its Endpoint hook may still compute a path — Azure's does. + // A row that named the vendor is refused; a row that named none and was + // matched by its URL is what the pre-catalog client served: an + // OpenAI-compatible endpoint at that URL. + if !resolved.Vendor.SupportsType(types.ModelTypeASR) { + if strings.TrimSpace(config.Provider) != "" { + return nil, fmt.Errorf("%s does not offer speech recognition in this build", resolved.Vendor.Name) + } + resolved, err = catalog.Resolve(catalog.Ref{ + Provider: catalog.GenericID, + Model: config.ModelName, + BaseURL: config.BaseURL, + ModelType: types.ModelTypeASR, + Extra: config.ExtraConfig, + }) + if err != nil { + return nil, err + } + } + if err := validateASRBaseURL(resolved.BaseURL); err != nil { + return nil, err + } + + vendor := resolved.Vendor + creds := catalog.Credentials{APIKey: config.APIKey} + if creds.APIKey == "" { + creds.APIKey = vendor.DefaultAPIKey + } + // Vendor headers first so a user header cannot silently replace a vendor + // beta flag, matching chat.NewRemoteChat. + headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders)) + for k, v := range vendor.Headers { + headers[k] = v + } + for k, v := range config.CustomHeaders { + headers[k] = v + } + settings := resolved.Transcriptions + endpoint := api.Endpoint{ + BaseURL: resolved.BaseURL, + Model: resolved.RemoteModel, + ModelID: config.ModelID, + Auth: vendor.AuthFunc(vendor.API, creds), + Headers: headers, + Client: newASRHTTPClient(time.Duration(settings.RequestTimeout) * time.Second), + } + if vendor.Endpoint != nil { + if url, query := vendor.Endpoint(catalog.EndpointRequest{ + BaseURL: resolved.BaseURL, + Model: resolved.RemoteModel, + ModelType: types.ModelTypeASR, + API: vendor.API, + Extra: config.ExtraConfig, + }); url != "" { + if err := validateASRBaseURL(url); err != nil { + return nil, err + } + endpoint.URL, endpoint.Query = url, query + } + } + + var client api.Transcriber + switch resolved.TranscriptionAPI { + case api.TranscriptionOpenAI: + client = openaitranscriptions.New(openaitranscriptions.Config{ + Endpoint: endpoint, Settings: settings, Retry: retryPolicy(), + }) + case api.TranscriptionChatAudio: + client = openaichataudio.New(openaichataudio.Config{ + Endpoint: endpoint, Settings: settings, Retry: retryPolicy(), + }) + default: + return nil, fmt.Errorf("unsupported transcription api %q for provider %s", + resolved.TranscriptionAPI, vendor.ID) + } + return &protocolASR{ + inner: client, + settings: settings, + vendor: vendor.Name, + endpoint: resolved.BaseURL, + modelName: config.ModelName, + modelID: config.ModelID, + }, nil +} + +// protocolASR adapts a protocol client to the ASR interface and refuses an +// upload the vendor has documented it will not take, before sending it. +type protocolASR struct { + inner api.Transcriber + settings catalog.TranscriptionsSettings + vendor string + endpoint string + modelName string + modelID string +} + +func (a *protocolASR) GetModelName() string { return a.modelName } +func (a *protocolASR) GetModelID() string { return a.modelID } + +func (a *protocolASR) Transcribe(ctx context.Context, audio []byte, fileName string) (*TranscriptionResult, error) { + if len(audio) == 0 { + return nil, fmt.Errorf("audio bytes are empty") + } + if limit := a.settings.MaxFileBytes; limit > 0 && len(audio) > limit { + return nil, fmt.Errorf("%s transcription: the audio is %.1f MB; %s accepts at most %d MB per file", + a.modelName, float64(len(audio))/(1<<20), a.vendor, limit>>20) + } + // The server identifies the format from the extension. + if fileName == "" { + fileName = "audio.mp3" + } + if formats := a.settings.Formats; len(formats) > 0 { + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(fileName)), ".") + if !slices.Contains(formats, ext) { + return nil, fmt.Errorf("%s transcription: %s accepts %s audio, not %q", + a.modelName, a.vendor, strings.Join(formats, "/"), fileName) + } + } + logger.Infof(ctx, "[ASR] transcribing model=%s endpoint=%s size=%d file=%s", + a.modelName, a.endpoint, len(audio), fileName) + + out, err := a.inner.Transcribe(ctx, api.TranscriptionRequest{ + Audio: audio, FileName: fileName, Language: languageFrom(ctx), + }) + if err != nil { + return nil, fmt.Errorf("ASR transcription request failed: %w", err) + } + result := &TranscriptionResult{Text: out.Text, Duration: out.Duration} + for _, s := range out.Segments { + result.Segments = append(result.Segments, Segment{Start: s.Start, End: s.End, Text: s.Text}) + } + logger.Infof(ctx, "[ASR] transcription completed, text length=%d", len(result.Text)) + return result, nil +} diff --git a/internal/models/asr/transport_security_test.go b/internal/models/asr/transport_security_test.go deleted file mode 100644 index a2d97025..00000000 --- a/internal/models/asr/transport_security_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package asr - -import ( - "testing" - - secutils "github.com/Tencent/WeKnora/internal/utils" -) - -func withASRSSRFWhitelist(t *testing.T, raw string) { - t.Helper() - t.Setenv("SSRF_WHITELIST", raw) - secutils.ResetSSRFWhitelistForTest() - t.Cleanup(secutils.ResetSSRFWhitelistForTest) -} - -func TestOpenAIASRRejectsInternalBaseURL(t *testing.T) { - withASRSSRFWhitelist(t, "") - - _, err := NewOpenAIASR(&Config{ - BaseURL: "http://169.254.169.254/latest/meta-data/", - ModelName: "asr-test", - }) - if err == nil { - t.Fatalf("NewOpenAIASR returned nil error for blocked internal BaseURL") - } -} diff --git a/internal/models/asr/wire_test.go b/internal/models/asr/wire_test.go new file mode 100644 index 00000000..503921fa --- /dev/null +++ b/internal/models/asr/wire_test.go @@ -0,0 +1,356 @@ +package asr + +import ( + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func allowLoopback(t *testing.T) { + t.Helper() + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + secutils.ResetSSRFWhitelistForTest() + t.Cleanup(secutils.ResetSSRFWhitelistForTest) +} + +type form struct { + path string + auth string + fields map[string]string + fileName string + language string // the language request header + // body is set instead of fields when the request was JSON: the + // chat-served recognisers take the audio as a data URI. + body map[string]any +} + +// upstream records the form it receives and answers the documented json +// shape, adding a segment when verbose_json was asked for. +func upstream(t *testing.T) (string, *form, *atomic.Int32) { + t.Helper() + allowLoopback(t) + got := &form{fields: map[string]string{}} + calls := &atomic.Int32{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + got.path, got.auth, got.language = r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("language") + mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + require.NoError(t, err) + if mediaType == "application/json" { + require.NoError(t, json.NewDecoder(r.Body).Decode(&got.body)) + _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":" hello "}}]}`)) + return + } + reader := multipart.NewReader(r.Body, params["boundary"]) + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err) + data, _ := io.ReadAll(part) + if part.FormName() == "file" { + got.fileName = part.FileName() + continue + } + got.fields[part.FormName()] = string(data) + } + if got.fields["response_format"] == "verbose_json" { + _, _ = w.Write([]byte(`{"text":" hello ","segments":[{"start":0,"end":1.5,"text":" hello "}]}`)) + return + } + _, _ = w.Write([]byte(`{"text":" hello "}`)) + })) + t.Cleanup(server.Close) + return server.URL, got, calls +} + +// TestTranscriptionWireFormatPerVendor pins the form each ASR vendor is sent. +// response_format appears only where the vendor documents it for that model: +// the pre-catalog client sent verbose_json to every one of them, which +// gpt-4o-transcribe rejects ("the only supported format is json") and which +// vox-box's FunASR backend answers with an undecodable bare string. +func TestTranscriptionWireFormatPerVendor(t *testing.T) { + chatAudio := func(model string) map[string]any { + return map[string]any{ + "model": model, + "messages": []any{map[string]any{ + "role": "user", + "content": []any{map[string]any{ + "type": "input_audio", + "input_audio": map[string]any{"data": "data:audio/wav;base64,UklGRg=="}, + }}, + }}, + } + } + cases := []struct { + name, provider, model, base string + wantPath string + wantFields map[string]string + wantBody map[string]any + wantSegments int + }{ + { + name: "openai whisper-1 serves segments", provider: "openai", model: "whisper-1", base: "/v1", + wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "whisper-1", "response_format": "verbose_json"}, + wantSegments: 1, + }, + { + name: "openai gpt-4o-transcribe accepts only json", provider: "openai", model: "gpt-4o-transcribe", + base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "gpt-4o-transcribe"}, + }, + { + name: "openai model not yet in the catalog", provider: "openai", model: "gpt-5-transcribe", + base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "gpt-5-transcribe"}, + }, + { + name: "siliconflow documents file and model only", provider: "siliconflow", + model: "FunAudioLLM/SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "FunAudioLLM/SenseVoiceSmall"}, + }, + { + name: "gpustack stays on json for its FunASR backend", provider: "gpustack", + model: "SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "SenseVoiceSmall"}, + }, + { + name: "generic stays on json", provider: "generic", model: "whisper-large-v3", + base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "whisper-large-v3"}, + }, + { + name: "zhipu glm-asr on the OpenAI shape", provider: "zhipu", model: "glm-asr-2512", + base: "/api/paas/v4", wantPath: "/api/paas/v4/audio/transcriptions", + wantFields: map[string]string{"model": "glm-asr-2512"}, + }, + { + name: "minimax on its own path", provider: "minimax", model: "asr-1.0", + base: "/v1", wantPath: "/v1/speech_to_text", + wantFields: map[string]string{"model": "asr-1.0"}, + }, + { + name: "openrouter", provider: "openrouter", model: "openai/whisper-large-v3", + base: "/api/v1", wantPath: "/api/v1/audio/transcriptions", + wantFields: map[string]string{"model": "openai/whisper-large-v3"}, + }, + { + name: "requesty whisper-1 serves segments", provider: "requesty", model: "openai/whisper-1", + base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "openai/whisper-1", "response_format": "verbose_json"}, + wantSegments: 1, + }, + { + name: "litellm proxy", provider: "litellm", model: "whisper", + base: "/v1", wantPath: "/v1/audio/transcriptions", + wantFields: map[string]string{"model": "whisper"}, + }, + { + name: "aliyun qwen3-asr-flash is served on chat", provider: "aliyun", model: "qwen3-asr-flash", + base: "/compatible-mode/v1", wantPath: "/compatible-mode/v1/chat/completions", + wantBody: chatAudio("qwen3-asr-flash"), + }, + { + name: "mimo asr is served on chat", provider: "mimo", model: "mimo-v2.5-asr", + base: "/v1", wantPath: "/v1/chat/completions", + wantBody: chatAudio("mimo-v2.5-asr"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + url, got, _ := upstream(t) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base, + ModelName: tc.model, APIKey: "k", + }) + require.NoError(t, err) + out, err := a.Transcribe(context.Background(), []byte("RIFF"), "meeting.wav") + require.NoError(t, err) + + assert.Equal(t, tc.wantPath, got.path) + assert.Equal(t, "Bearer k", got.auth) + if tc.wantBody != nil { + assert.Equal(t, tc.wantBody, got.body) + } else { + assert.Equal(t, tc.wantFields, got.fields) + assert.Equal(t, "meeting.wav", got.fileName) + } + assert.Equal(t, "hello", out.Text) + assert.Len(t, out.Segments, tc.wantSegments) + }) + } +} + +// A file over the documented ceiling is refused before it is uploaded: the +// vendor would reject it anyway, after the whole upload. +func TestOversizedAudioIsRefusedBeforeUpload(t *testing.T) { + url, _, calls := upstream(t) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1", + ModelName: "whisper-1", APIKey: "k", + }) + require.NoError(t, err) + + _, err = a.Transcribe(context.Background(), make([]byte, 25<<20+1), "long.mp3") + require.Error(t, err) + assert.Contains(t, err.Error(), "at most 25 MB") + assert.Zero(t, calls.Load(), "nothing should have been sent") + + _, err = a.Transcribe(context.Background(), make([]byte, 25<<20), "exactly.mp3") + require.NoError(t, err, "the ceiling itself is allowed") +} + +// The chat-served recognisers cap the data URI as sent at 10 MB, prefix +// included, so three quarters of 10 MB of audio is already too much. +func TestChatServedCeilingCountsTheWholeDataURI(t *testing.T) { + url, _, calls := upstream(t) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "aliyun", BaseURL: url + "/compatible-mode/v1", + ModelName: "qwen3-asr-flash", APIKey: "k", + }) + require.NoError(t, err) + + _, err = a.Transcribe(context.Background(), make([]byte, 10<<20*3/4), "long.wav") + require.Error(t, err) + assert.Zero(t, calls.Load()) + + prefix := len("data:audio/wav;base64,") + _, err = a.Transcribe(context.Background(), make([]byte, (10<<20-prefix)/4*3), "fits.wav") + require.NoError(t, err) +} + +// Only qwen3-asr-flash takes the audio in the request. Alibaba's other +// recognition models want a public URL or an asynchronous task, so a row +// naming one is refused with the reason rather than sent a chat request. +func TestAlibabaRecognitionModelsOtherThanQwenASRFlashAreRefused(t *testing.T) { + _, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "aliyun", + BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", ModelName: "paraformer-v2", APIKey: "k", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "public file URL") +} + +// A vendor that does not declare speech recognition is not trusted with it. +// Azure's Endpoint hook still knows a transcription path, but nobody has +// verified it; a row that names Azure is refused, and a row that names no +// vendor gets what the pre-catalog client sent: the OpenAI shape at its URL. +func TestVendorsWithoutASRAreNotRoutedThroughTheirHooks(t *testing.T) { + _, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "azure_openai", + BaseURL: "https://example.openai.azure.com", ModelName: "whisper", APIKey: "k", + }) + require.Error(t, err) + + // Detection matches URL patterns as substrings, so a path carrying Azure's + // host detects as Azure while the request still reaches the test server. + url, got, _ := upstream(t) + base := url + "/openai.azure.com/v1" + require.Equal(t, "azure_openai", catalog.DetectByURL(base)) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, BaseURL: base, ModelName: "whisper", APIKey: "k", + }) + require.NoError(t, err) + _, err = a.Transcribe(context.Background(), []byte("RIFF"), "a.wav") + require.NoError(t, err) + assert.Equal(t, "/openai.azure.com/v1/audio/transcriptions", got.path, "not Azure's /openai/v1 path") + assert.Equal(t, "Bearer k", got.auth, "not Azure's api-key header") +} + +// A format the vendor does not list is refused before upload. +func TestUndocumentedFormatIsRefusedBeforeUpload(t *testing.T) { + url, _, calls := upstream(t) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "zhipu", BaseURL: url + "/api/paas/v4", + ModelName: "glm-asr-2512", APIKey: "k", + }) + require.NoError(t, err) + _, err = a.Transcribe(context.Background(), []byte("x"), "memo.m4a") + require.Error(t, err) + assert.Contains(t, err.Error(), "wav/mp3") + assert.Zero(t, calls.Load()) +} + +// The knowledge base's language hint reaches each vendor where its reference +// puts it, and nowhere for a vendor that documents none. +func TestLanguageHintGoesWhereEachVendorDocumentsIt(t *testing.T) { + for _, tc := range []struct { + name, provider, model, base string + check func(t *testing.T, got *form) + }{ + {"openai form field", "openai", "whisper-1", "/v1", func(t *testing.T, got *form) { + assert.Equal(t, "zh", got.fields["language"]) + assert.Empty(t, got.language) + }}, + {"minimax header", "minimax", "asr-1.0", "/v1", func(t *testing.T, got *form) { + assert.Equal(t, "zh", got.language) + assert.NotContains(t, got.fields, "language") + }}, + {"aliyun asr_options", "aliyun", "qwen3-asr-flash", "/compatible-mode/v1", func(t *testing.T, got *form) { + assert.Equal(t, map[string]any{"language": "zh"}, got.body["asr_options"]) + }}, + { + "siliconflow documents none", "siliconflow", "FunAudioLLM/SenseVoiceSmall", "/v1", + func(t *testing.T, got *form) { + assert.NotContains(t, got.fields, "language") + assert.Empty(t, got.language) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + url, got, _ := upstream(t) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base, + ModelName: tc.model, APIKey: "k", + }) + require.NoError(t, err) + _, err = a.Transcribe(WithLanguage(context.Background(), "zh"), []byte("RIFF"), "a.wav") + require.NoError(t, err) + tc.check(t, got) + }) + } + + // "auto" is every vendor's default and is not sent. + url, got, _ := upstream(t) + a, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1", ModelName: "whisper-1", APIKey: "k", + }) + require.NoError(t, err) + _, err = a.Transcribe(WithLanguage(context.Background(), "auto"), []byte("RIFF"), "a.wav") + require.NoError(t, err) + assert.NotContains(t, got.fields, "language") +} + +func TestEmptyAudioIsAnError(t *testing.T) { + url, _, calls := upstream(t) + a, err := NewASR(&Config{Source: types.ModelSourceRemote, Provider: "generic", BaseURL: url, ModelName: "m"}) + require.NoError(t, err) + _, err = a.Transcribe(context.Background(), nil, "a.wav") + assert.Error(t, err) + assert.Zero(t, calls.Load()) +} + +func TestInternalBaseURLIsRefused(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "") + secutils.ResetSSRFWhitelistForTest() + t.Cleanup(secutils.ResetSSRFWhitelistForTest) + _, err := NewASR(&Config{ + Source: types.ModelSourceRemote, Provider: "generic", + BaseURL: "http://169.254.169.254/latest/meta-data/", ModelName: "asr-test", + }) + assert.Error(t, err) +} diff --git a/internal/models/catalog/compat.go b/internal/models/catalog/compat.go index 234d8a2b..f2a479d0 100644 --- a/internal/models/catalog/compat.go +++ b/internal/models/catalog/compat.go @@ -346,6 +346,8 @@ type VendorCompat struct { Rerank RerankCompat `json:"rerank,omitempty"` // Embeddings is the embedding protocol overlay, likewise one per vendor. Embeddings EmbeddingsCompat `json:"embeddings,omitempty"` + // Transcriptions is the speech-to-text overlay, likewise one per vendor. + Transcriptions TranscriptionsCompat `json:"transcriptions,omitempty"` } // apply writes the set fields of an overlay struct onto a settings struct diff --git a/internal/models/catalog/registry.go b/internal/models/catalog/registry.go index 91767fde..fc132a49 100644 --- a/internal/models/catalog/registry.go +++ b/internal/models/catalog/registry.go @@ -34,6 +34,9 @@ func Register(v *Vendor) { if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) { v.RerankAPI = api.RerankCohere } + if v.TranscriptionAPI == "" && v.SupportsType(types.ModelTypeASR) { + v.TranscriptionAPI = api.TranscriptionOpenAI + } if v.EmbeddingAPI == "" && v.SupportsType(types.ModelTypeEmbedding) { v.EmbeddingAPI = api.EmbeddingOpenAI } diff --git a/internal/models/catalog/resolve.go b/internal/models/catalog/resolve.go index ef620227..fa8542c2 100644 --- a/internal/models/catalog/resolve.go +++ b/internal/models/catalog/resolve.go @@ -81,6 +81,11 @@ type Resolved struct { // for the same reason. EmbeddingAPI api.EmbeddingAPI Embeddings EmbeddingsSettings + + // TranscriptionAPI and Transcriptions are filled only for an ASR + // reference. + TranscriptionAPI api.TranscriptionAPI + Transcriptions TranscriptionsSettings } // Resolve merges the vendor, catalog entry, extra-config and per-row @@ -122,6 +127,9 @@ func Resolve(ref Ref) (*Resolved, error) { if modelType == types.ModelTypeEmbedding { return resolveEmbeddings(ref, vendor, spec, cataloged, baseURL) } + if modelType == types.ModelTypeASR { + return resolveTranscriptions(ref, vendor, spec, cataloged, baseURL) + } resolvedAPI := spec.API if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" { @@ -537,3 +545,62 @@ func resolveEmbeddings( } return out, nil } + +// resolveTranscriptions merges the speech-to-text layers, in the same order +// as resolveEmbeddings: protocol default, vendor compat, the catalog entry, +// the row's own compat, then remote_model_name. +func resolveTranscriptions( + ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string, +) (*Resolved, error) { + protocol := vendor.TranscriptionAPI + if protocol == "" { + protocol = api.TranscriptionOpenAI + } + settings := DefaultTranscriptions() + apply(&settings, &vendor.Compat.Transcriptions) + for _, raw := range []json.RawMessage{spec.Compat, ref.Override.CompatJSON()} { + if len(raw) == 0 { + continue + } + overlay := &TranscriptionsCompat{} + if err := decodeCompat(raw, overlay); err != nil { + return nil, fmt.Errorf("transcriptions compat: %w", err) + } + apply(&settings, overlay) + } + if settings.API != "" { + protocol = settings.API + } + if !protocol.Known() { + return nil, fmt.Errorf("catalog: unknown transcription api %q for %s/%s", protocol, vendor.ID, spec.ID) + } + settings.API = protocol + switch settings.LanguageParam { + case "", LanguageForm, LanguageHeader, LanguageASROptions: + default: + return nil, fmt.Errorf("catalog: unknown language_param %q for %s/%s", + settings.LanguageParam, vendor.ID, spec.ID) + } + // Checked after every layer: an entry may lift a vendor-wide refusal for + // the one model that takes the audio in the request. + if settings.UnsupportedReason != "" { + return nil, fmt.Errorf( + "catalog: %s does not serve %q through a protocol this build implements: %s", + vendor.ID, spec.ID, settings.UnsupportedReason, + ) + } + + out := &Resolved{ + Vendor: vendor, + Spec: spec, + Cataloged: cataloged, + BaseURL: baseURL, + RemoteModel: ref.Model, + TranscriptionAPI: protocol, + Transcriptions: settings, + } + if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" { + out.RemoteModel = override + } + return out, nil +} diff --git a/internal/models/catalog/transcriptions.go b/internal/models/catalog/transcriptions.go new file mode 100644 index 00000000..0a7b74ce --- /dev/null +++ b/internal/models/catalog/transcriptions.go @@ -0,0 +1,70 @@ +package catalog + +import "github.com/Tencent/WeKnora/internal/models/api" + +// Where a transcription protocol puts the operator's language hint. Each +// vendor documents one place, or none. +const ( + // LanguageForm is a multipart field named language (OpenAI's shape, + // ISO-639-1). + LanguageForm = "form" + // LanguageHeader is an HTTP request header named language (MiniMax, + // BCP-47). + LanguageHeader = "header" + // LanguageASROptions is asr_options.language in a chat-served request + // body (Alibaba, Xiaomi). + LanguageASROptions = "asr_options" +) + +// TranscriptionsCompat is the overlay form (every field optional) of the +// speech-to-text settings. JSON keys are the names used in models.json. +type TranscriptionsCompat struct { + API *api.TranscriptionAPI `json:"api,omitempty"` + Path *string `json:"path,omitempty"` + // ResponseFormat is sent as response_format when non-empty. OpenAI's + // gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json, which is + // also the default, so most models leave it unset. + ResponseFormat *string `json:"response_format,omitempty"` + // LanguageParam names where the language hint goes: form, header or + // asr_options; empty sends none. + LanguageParam *string `json:"language_param,omitempty"` + // MaxFileBytes is the documented upload ceiling for the audio itself. + MaxFileBytes *int `json:"max_file_bytes,omitempty"` + // MaxEncodedBytes is the documented ceiling on the base64 data URI, for + // the protocols that send the audio that way. + MaxEncodedBytes *int `json:"max_encoded_bytes,omitempty"` + // Formats lists the file extensions the vendor documents, without the + // dot. Empty means the vendor documents no closed list. + Formats []string `json:"formats,omitempty"` + RequestTimeout *int `json:"request_timeout_seconds,omitempty"` + // UnsupportedReason marks a model this build cannot call. + UnsupportedReason *string `json:"unsupported_reason,omitempty"` +} + +// TranscriptionsSettings is the resolved (fully defaulted) form. +type TranscriptionsSettings struct { + API api.TranscriptionAPI + Path string + ResponseFormat string + LanguageParam string + // MaxFileBytes and MaxEncodedBytes refuse an upload the vendor would + // reject anyway, before sending it; 0 leaves the check to the vendor. + MaxFileBytes int + MaxEncodedBytes int + Formats []string + // RequestTimeout caps one request, in seconds. + RequestTimeout int + // UnsupportedReason is set where a vendor's recognition models, or some + // of them, cannot be called with the audio in the request — they take a + // public URL or run as asynchronous tasks. Resolve refuses such a model + // rather than sending it a shape it does not accept. + UnsupportedReason string +} + +// DefaultTranscriptions is the protocol baseline: a form with file and model +// and nothing else, which every speech-to-text endpoint in this catalog +// documents. The deadline is the one the pre-catalog client carried; audio +// transcription is slow. +func DefaultTranscriptions() TranscriptionsSettings { + return TranscriptionsSettings{Path: "/audio/transcriptions", RequestTimeout: 300} +} diff --git a/internal/models/catalog/types.go b/internal/models/catalog/types.go index 90856f42..bc2985be 100644 --- a/internal/models/catalog/types.go +++ b/internal/models/catalog/types.go @@ -251,6 +251,9 @@ type Vendor struct { // OpenAI shape for any vendor that serves embeddings without naming // another. EmbeddingAPI api.EmbeddingAPI + // TranscriptionAPI is the speech-to-text protocol. Register defaults it + // to the OpenAI shape for any vendor that serves ASR. + TranscriptionAPI api.TranscriptionAPI // DefaultBaseURLs by model type; GetDefaultURL falls back to chat. DefaultBaseURLs map[types.ModelType]string ModelTypes []types.ModelType diff --git a/internal/models/catalog/validate.go b/internal/models/catalog/validate.go index eaff1ce6..5955ae8e 100644 --- a/internal/models/catalog/validate.go +++ b/internal/models/catalog/validate.go @@ -14,10 +14,9 @@ import ( // protocol, a bad compat key or an invalid thinking level is rejected where // it is written instead of surfacing as a failed chat call much later. // -// Chat, VLM, embedding and rerank rows are checked; all four are built from -// catalog.Resolve, so a row that does not resolve cannot be used. ASR rows -// return nil: the ASR client does not resolve through the catalog yet, so a -// spec stored on such a row is inert — neither validated nor applied. +// Every model type is checked: all of them are built from catalog.Resolve, +// so a row that does not resolve cannot be used and should fail the save +// rather than the first call. func ValidateRow(modelName string, modelType types.ModelType, params *types.ModelParameters) error { if params == nil || !resolvesThroughCatalog(modelType) { return nil @@ -44,7 +43,8 @@ func ValidateRow(modelName string, modelType types.ModelType, params *types.Mode func resolvesThroughCatalog(modelType types.ModelType) bool { switch modelType { - case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank: + case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank, + types.ModelTypeASR: return true } return false diff --git a/internal/models/catalog/validate_test.go b/internal/models/catalog/validate_test.go index 0e6186f9..68034a96 100644 --- a/internal/models/catalog/validate_test.go +++ b/internal/models/catalog/validate_test.go @@ -56,9 +56,17 @@ func TestValidateRow(t *testing.T) { wantErr: true, }, { - name: "asr rows are not resolved through the catalog", modelName: "whisper-1", + name: "a catalogued asr row resolves", modelName: "whisper-1", modelType: types.ModelTypeASR, - params: &types.ModelParameters{Provider: "openai", Spec: &types.ModelSpecOverride{API: "nope"}}, + params: &types.ModelParameters{Provider: "openai"}, + }, + { + name: "unknown asr compat key is rejected", modelName: "whisper-1", + modelType: types.ModelTypeASR, + params: &types.ModelParameters{ + Provider: "openai", Spec: &types.ModelSpecOverride{Compat: map[string]any{"dimensions_field": "x"}}, + }, + wantErr: true, }, { name: "a catalogued chat row resolves", modelName: "deepseek-v4-pro", diff --git a/internal/models/vendors/aliyun/models.json b/internal/models/vendors/aliyun/models.json index bff7b78f..fd06a04d 100644 --- a/internal/models/vendors/aliyun/models.json +++ b/internal/models/vendors/aliyun/models.json @@ -55,6 +55,8 @@ {"name": "qwen2.5-vl-embedding (fused vectors only)", "match": "qwen2.5-vl-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "dimension", "max_batch_size": 1}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"}, {"name": "multimodal-embedding-v1", "match": "multimodal-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "", "max_batch_size": 20}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"}, {"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}, - {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"} + {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}, + {"id": "qwen3-asr-flash", "name": "Qwen3 ASR Flash", "type": "ASR", "compat": {"api": "openai-chat-audio", "max_encoded_bytes": 10485760, "language_param": "asr_options"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"}, + {"name": "Other Alibaba recognition models (not callable with the audio in the request)", "match": "*", "type": "ASR", "compat": {"unsupported_reason": "Alibaba's other recognition models take a public file URL or run as asynchronous tasks; only qwen3-asr-flash accepts the audio in the request"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"} ] } diff --git a/internal/models/vendors/aliyun/vendor.go b/internal/models/vendors/aliyun/vendor.go index 36d9da82..9c567258 100644 --- a/internal/models/vendors/aliyun/vendor.go +++ b/internal/models/vendors/aliyun/vendor.go @@ -136,6 +136,7 @@ func init() { types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, RerankAPI: api.RerankDashScope, // Text and multimodal embeddings live under different roots of the @@ -160,6 +161,13 @@ func init() { return root + "/compatible-mode/v1/embeddings", nil }, Compat: catalog.VendorCompat{ + // ASR: qwen3-asr-flash is the one recognition model callable with + // the audio in the request, on the compatible chat endpoint as a + // base64 data URI; its catalog entry declares that. Every other + // ASR name falls to a catch-all entry that refuses it: Paraformer, + // Fun-ASR and the *-filetrans models are asynchronous tasks that + // take a public file URL, which no protocol here can send + // (https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference). // Text models: the OpenAI-compatible endpoint // (https://help.aliyun.com/zh/model-studio/embedding-interfaces-compatible-with-openai), // which takes model, input, dimensions and encoding_format. diff --git a/internal/models/vendors/azure_openai/vendor.go b/internal/models/vendors/azure_openai/vendor.go index 8c3e3418..63e7909c 100644 --- a/internal/models/vendors/azure_openai/vendor.go +++ b/internal/models/vendors/azure_openai/vendor.go @@ -124,14 +124,16 @@ func init() { types.ModelTypeEmbedding: BaseURL, types.ModelTypeVLLM: BaseURL, }, - // ASR is deliberately absent: Azure serves Whisper, but - // internal/models/asr builds every remote client with a plain - // OpenAI config (Bearer auth, {base}/audio/transcriptions) and has - // no provider or extra-config field to route through this vendor's - // api-key header and deployments path. Advertising the type would - // let the picker create a row that fails at first use. The Endpoint - // hook below already knows the ASR path, so re-adding the type is a - // one-line change once asr.Config carries the provider. + // ASR is deliberately absent. The ASR client now resolves through + // this vendor (api-key header, Endpoint hook below), but the only + // v1 reference for audio is the preview one — + // POST {endpoint}/openai/v1/audio/transcriptions?api-version=preview + // (https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest) + // — while the hook sends a row without api_version to the v1 path + // with no version at all. Until that is verified against a live + // resource, advertising the type would let the picker create a row + // that may fail at first use. A row with an explicit api_version + // would take the documented deployments path. ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, diff --git a/internal/models/vendors/generic/vendor.go b/internal/models/vendors/generic/vendor.go index 8b7a0424..145530fd 100644 --- a/internal/models/vendors/generic/vendor.go +++ b/internal/models/vendors/generic/vendor.go @@ -108,6 +108,17 @@ func init() { ModelTypes: []types.ModelType{types.ModelTypeRerank}, }}, Compat: catalog.VendorCompat{ + // Transcriptions keeps the baseline, json by default. The + // pre-catalog client always asked for verbose_json, which not every + // model or server can produce: OpenAI's gpt-4o transcribers accept + // only json, and vox-box's FunASR backend answers verbose_json with + // a bare JSON string. A row that wants segments sets + // {"response_format": "verbose_json"} in its compat. The language + // hint goes as OpenAI's language form field, which a server that + // does not implement it ignores. + Transcriptions: catalog.TranscriptionsCompat{ + LanguageParam: catalog.Ptr(catalog.LanguageForm), + }, Embeddings: catalog.EmbeddingsCompat{ // Whatever the operator runs. Everything the pre-catalog client // sent stays: vLLM, SGLang, TEI and Ollama's OpenAI route all diff --git a/internal/models/vendors/gpustack/vendor.go b/internal/models/vendors/gpustack/vendor.go index 89f24dc0..f42f3ff1 100644 --- a/internal/models/vendors/gpustack/vendor.go +++ b/internal/models/vendors/gpustack/vendor.go @@ -118,6 +118,17 @@ func init() { ModelTypes: []types.ModelType{types.ModelTypeRerank}, }}, Compat: catalog.VendorCompat{ + // Transcriptions keeps the baseline, json by default. GPUStack + // serves audio through vox-box (https://github.com/gpustack/vox-box), + // whose route wraps a json answer as {"text": ...} but returns + // verbose_json as whatever the backend produced — and its FunASR + // backend (SenseVoice, Paraformer) produces a bare string for every + // format. The pre-catalog client's verbose_json therefore arrived + // as a JSON string that could not be decoded. The route reads a + // language form field. + Transcriptions: catalog.TranscriptionsCompat{ + LanguageParam: catalog.Ptr(catalog.LanguageForm), + }, Embeddings: catalog.EmbeddingsCompat{ // vLLM's embedding server: dimensions for Matryoshka models, the // row's truncation budget, and encoding_format. diff --git a/internal/models/vendors/litellm/vendor.go b/internal/models/vendors/litellm/vendor.go index 02d0d96b..23c24f2a 100644 --- a/internal/models/vendors/litellm/vendor.go +++ b/internal/models/vendors/litellm/vendor.go @@ -77,6 +77,7 @@ func init() { types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, ExtraFields: []catalog.ExtraField{{ Key: catalog.ExtraScoreScale, @@ -104,6 +105,9 @@ func init() { ModelTypes: []types.ModelType{types.ModelTypeRerank}, }}, Compat: catalog.VendorCompat{ + // Transcriptions keeps the baseline: the proxy serves the OpenAI + // /audio/transcriptions route for whichever upstream it is + // configured with (https://docs.litellm.ai/docs/audio_transcription). Embeddings: catalog.EmbeddingsCompat{ // https://docs.litellm.ai/docs/embedding/supported_embedding: // model, input, user, dimensions, encoding_format; anything else diff --git a/internal/models/vendors/mimo/models.json b/internal/models/vendors/mimo/models.json index c54858e3..b2d86cd2 100644 --- a/internal/models/vendors/mimo/models.json +++ b/internal/models/vendors/mimo/models.json @@ -5,6 +5,7 @@ {"id": "mimo-v2.5-pro", "name": "MiMo V2.5 Pro", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.435, "output": 0.87, "cache_read": 0.0036}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"}, {"id": "mimo-v2.5", "name": "MiMo V2.5", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.14, "output": 0.28, "cache_read": 0.0028}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"}, {"id": "mimo-v2.5-pro-ultraspeed", "name": "MiMo V2.5 Pro Ultraspeed", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 1.305, "output": 2.61, "cache_read": 0.0108}, "source": "https://mimo.mi.com/static/docs/news/latest/beta-extended.md"}, - {"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"} + {"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"}, + {"id": "mimo-v2.5-asr", "name": "MiMo V2.5 ASR", "type": "ASR", "source": "https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition"} ] } diff --git a/internal/models/vendors/mimo/vendor.go b/internal/models/vendors/mimo/vendor.go index f3c7ec13..48ece1f1 100644 --- a/internal/models/vendors/mimo/vendor.go +++ b/internal/models/vendors/mimo/vendor.go @@ -71,10 +71,22 @@ func init() { DefaultBaseURLs: map[types.ModelType]string{ types.ModelTypeKnowledgeQA: BaseURL, }, + TranscriptionAPI: api.TranscriptionChatAudio, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, + types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + // ASR: mimo-v2.5-asr on the chat endpoint, WAV or MP3 as a base64 + // data URI whose "encoded string size must not exceed 10 MB" + // (https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition). + Transcriptions: catalog.TranscriptionsCompat{ + // The ceiling counts the whole data URI as sent. + MaxEncodedBytes: catalog.Ptr(10 << 20), + Formats: []string{"wav", "mp3"}, + // asr_options.language: "auto|zh|en". + LanguageParam: catalog.Ptr(catalog.LanguageASROptions), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType), PromptCacheAccounting: catalog.Ptr(true), diff --git a/internal/models/vendors/minimax/models.json b/internal/models/vendors/minimax/models.json index 7aababf4..b518a690 100644 --- a/internal/models/vendors/minimax/models.json +++ b/internal/models/vendors/minimax/models.json @@ -9,6 +9,7 @@ {"id": "MiniMax-M2.5-highspeed", "name": "MiniMax M2.5 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.6, "output": 2.4, "cache_read": 0.06, "cache_write": 0.375}, "thinking_levels": {"off": null}}, {"id": "MiniMax-M2.1", "name": "MiniMax M2.1", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.03, "cache_write": 0.375}, "thinking_levels": {"off": null}}, {"id": "MiniMax-M2.1-highspeed", "name": "MiniMax M2.1 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "thinking_levels": {"off": null}, "source": "https://platform.minimax.io/docs/guides/text-generation"}, - {"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}} + {"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}}, + {"id": "asr-1.0", "name": "MiniMax ASR 1.0", "type": "ASR", "source": "https://platform.minimax.io/docs/api-reference/speech-to-text"} ] } diff --git a/internal/models/vendors/minimax/vendor.go b/internal/models/vendors/minimax/vendor.go index 08652c12..d5a6f156 100644 --- a/internal/models/vendors/minimax/vendor.go +++ b/internal/models/vendors/minimax/vendor.go @@ -95,8 +95,23 @@ func init() { }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, + types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + Transcriptions: catalog.TranscriptionsCompat{ + // https://platform.minimax.io/docs/api-reference/speech-to-text + // (and the same page on platform.minimax.cn): multipart model + // (asr-1.0) + file on /v1/speech_to_text rather than the OpenAI + // path, answering {text, duration, trace_id} for the default + // json. At most 50 MB and 500 seconds; "超出会返回 400 而不会被截断". + Path: catalog.Ptr("/speech_to_text"), + MaxFileBytes: catalog.Ptr(50 << 20), + // "wav / aiff / flac / alac(m4a) / mp3 / aac / opus / ogg"; + // "不支持无容器的裸 PCM 数据". The language hint is a request + // header, a BCP-47 tag, not a form field. + Formats: []string{"wav", "aiff", "flac", "m4a", "mp3", "aac", "opus", "ogg"}, + LanguageParam: catalog.Ptr(catalog.LanguageHeader), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType), ThinkingEnabledValue: catalog.Ptr("adaptive"), diff --git a/internal/models/vendors/openai/models.json b/internal/models/vendors/openai/models.json index be7775ce..b6710376 100644 --- a/internal/models/vendors/openai/models.json +++ b/internal/models/vendors/openai/models.json @@ -29,6 +29,10 @@ {"id": "text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8191, "cost": {"input": 0.02, "output": 0}}, {"id": "text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8191, "cost": {"input": 0.13, "output": 0}}, + {"id": "whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"}, + {"id": "gpt-transcribe", "name": "GPT Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"}, + {"id": "gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"}, + {"id": "gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"}, {"id": "text-embedding-ada-002", "name": "text-embedding-ada-002", "type": "Embedding", "dimension": 1536, "context_window": 8192, "cost": {"input": 0.1, "output": 0}, "compat": {"dimensions_field": ""}} ] } diff --git a/internal/models/vendors/openai/vendor.go b/internal/models/vendors/openai/vendor.go index a6f99249..d1d02d53 100644 --- a/internal/models/vendors/openai/vendor.go +++ b/internal/models/vendors/openai/vendor.go @@ -97,6 +97,19 @@ func init() { types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + Transcriptions: catalog.TranscriptionsCompat{ + // The transcription reference (URL in openaitranscriptions): + // response_format differs per model — "For gpt-4o-transcribe + // and gpt-4o-mini-transcribe, the only supported format is + // json", the default — so it is declared per entry, not here. + // "Files can be up to 25 MB" + // (https://developers.openai.com/api/docs/guides/speech-to-text). + MaxFileBytes: catalog.Ptr(25 << 20), + // "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm", and a + // language field in ISO-639-1. + Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"}, + LanguageParam: catalog.Ptr(catalog.LanguageForm), + }, Embeddings: catalog.EmbeddingsCompat{ // https://developers.openai.com/api/reference/resources/embeddings/methods/create: // model, input, dimensions (text-embedding-3 and later; the ada-002 diff --git a/internal/models/vendors/openrouter/models.json b/internal/models/vendors/openrouter/models.json index a76f2d05..23660cd7 100644 --- a/internal/models/vendors/openrouter/models.json +++ b/internal/models/vendors/openrouter/models.json @@ -19,6 +19,7 @@ {"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}}, {"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"}, {"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"}, - {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"} + {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}, + {"id": "openai/whisper-large-v3", "name": "Whisper Large v3", "type": "ASR", "source": "https://openrouter.ai/docs/api/api-reference/stt/create-transcription"} ] } diff --git a/internal/models/vendors/openrouter/vendor.go b/internal/models/vendors/openrouter/vendor.go index c475ea4e..38d62c26 100644 --- a/internal/models/vendors/openrouter/vendor.go +++ b/internal/models/vendors/openrouter/vendor.go @@ -75,8 +75,18 @@ func init() { types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + Transcriptions: catalog.TranscriptionsCompat{ + // https://openrouter.ai/docs/api/api-reference/stt/create-transcription: + // OpenAI-style multipart file + model, json by default. "Max 25 + // MB; send larger files as base64 JSON via input_audio." + MaxFileBytes: catalog.Ptr(25 << 20), + // language is an ISO-639-1 form field. The format "is derived + // from the filename extension", with no closed list. + LanguageParam: catalog.Ptr(catalog.LanguageForm), + }, Embeddings: catalog.EmbeddingsCompat{ // https://openrouter.ai/docs/api/api-reference/embeddings/create-embeddings: // model, input, dimensions, encoding_format, input_type, provider, diff --git a/internal/models/vendors/requesty/models.json b/internal/models/vendors/requesty/models.json index 360c6a99..c1d2d27e 100644 --- a/internal/models/vendors/requesty/models.json +++ b/internal/models/vendors/requesty/models.json @@ -12,6 +12,9 @@ {"id": "google/gemini-3.6-flash", "name": "Gemini 3.6 Flash", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 65536, "thinking_levels": {"off": null, "minimal": "min", "xhigh": "high", "max": "high"}}, {"id": "deepseek/deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro", "reasoning": true, "input": ["text"], "context_window": 1000000, "max_output_tokens": 131072}, {"id": "zai/glm-5.2", "name": "GLM-5.2", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072}, - {"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144} + {"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144}, + {"id": "openai/gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"}, + {"id": "openai/gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"}, + {"id": "openai/whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"} ] } diff --git a/internal/models/vendors/requesty/vendor.go b/internal/models/vendors/requesty/vendor.go index 283f44c8..74be132d 100644 --- a/internal/models/vendors/requesty/vendor.go +++ b/internal/models/vendors/requesty/vendor.go @@ -77,8 +77,19 @@ func init() { types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, types.ModelTypeVLLM, + types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + Transcriptions: catalog.TranscriptionsCompat{ + // https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create: + // OpenAI-compatible multipart, json by default, verbose_json + // available. "The maximum upload size per request is 32 MB." + MaxFileBytes: catalog.Ptr(32 << 20), + // "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, webm"; language + // is an ISO 639-1 form field. + Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"}, + LanguageParam: catalog.Ptr(catalog.LanguageForm), + }, Embeddings: catalog.EmbeddingsCompat{ // https://docs.requesty.ai/api-reference/endpoint/embeddings-create: // model, input, encoding_format, dimensions. diff --git a/internal/models/vendors/siliconflow/models.json b/internal/models/vendors/siliconflow/models.json index e51084c6..44cb98d2 100644 --- a/internal/models/vendors/siliconflow/models.json +++ b/internal/models/vendors/siliconflow/models.json @@ -24,6 +24,7 @@ {"id": "Qwen/Qwen3-Embedding-0.6B", "name": "Qwen3 Embedding 0.6B", "type": "Embedding", "dimension": 1024, "source": "https://api-docs.siliconflow.cn/docs/api/embeddings-post"}, {"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker v2 M3", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"}, {"id": "Qwen/Qwen3-Reranker-8B", "name": "Qwen3 Reranker 8B", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"}, - {"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://docs.siliconflow.cn/cn/api-reference/audio/create-audio-transcriptions"} + {"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"}, + {"id": "TeleAI/TeleSpeechASR", "name": "TeleSpeechASR", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"} ] } diff --git a/internal/models/vendors/siliconflow/vendor.go b/internal/models/vendors/siliconflow/vendor.go index f332e6fc..1421d5d7 100644 --- a/internal/models/vendors/siliconflow/vendor.go +++ b/internal/models/vendors/siliconflow/vendor.go @@ -77,6 +77,12 @@ func init() { types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + Transcriptions: catalog.TranscriptionsCompat{ + // https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post: + // file and model only — no response_format — and a file of + // "时长不超过 1 小时,文件大小不超过 50MB". + MaxFileBytes: catalog.Ptr(50 << 20), + }, Embeddings: catalog.EmbeddingsCompat{ // https://api-docs.siliconflow.cn/docs/api/embeddings-post: model, // input, encoding_format, and dimensions "仅 Qwen/Qwen3 系列支持" diff --git a/internal/models/vendors/vendors_test.go b/internal/models/vendors/vendors_test.go index 452e6c39..f3ebb50f 100644 --- a/internal/models/vendors/vendors_test.go +++ b/internal/models/vendors/vendors_test.go @@ -110,6 +110,9 @@ func TestEveryCatalogEntryResolves(t *testing.T) { if modelType == types.ModelTypeKnowledgeQA && !r.API.Known() { t.Errorf("%s/%s: resolved to unknown API %q", id, name, r.API) } + if modelType == types.ModelTypeASR && !r.TranscriptionAPI.Known() { + t.Errorf("%s/%s: resolved to unknown transcription API %q", id, name, r.TranscriptionAPI) + } if modelType == types.ModelTypeEmbedding && !r.EmbeddingAPI.Known() { t.Errorf("%s/%s: resolved to unknown embedding API %q", id, name, r.EmbeddingAPI) } diff --git a/internal/models/vendors/zhipu/models.json b/internal/models/vendors/zhipu/models.json index 62c15a1e..bd2ea0e3 100644 --- a/internal/models/vendors/zhipu/models.json +++ b/internal/models/vendors/zhipu/models.json @@ -15,6 +15,7 @@ {"id": "glm-4.5-air", "name": "GLM-4.5 Air", "reasoning": true, "input": ["text"], "context_window": 131072, "max_output_tokens": 98304, "cost": {"input": 0.2, "output": 1.1, "cache_read": 0.03}, "compat": {"supports_reasoning_effort": false}}, {"id": "embedding-3", "name": "Embedding-3", "type": "Embedding", "dimension": 2048, "compat": {"max_batch_size": 64}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"}, {"id": "embedding-2", "name": "Embedding-2", "type": "Embedding", "dimension": 1024, "compat": {"dimensions_field": ""}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"}, - {"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"} + {"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"}, + {"id": "glm-asr-2512", "name": "GLM-ASR-2512 (at most 30 s per file)", "type": "ASR", "source": "https://docs.bigmodel.cn/api-reference/模型-api/语音转文本"} ] } diff --git a/internal/models/vendors/zhipu/vendor.go b/internal/models/vendors/zhipu/vendor.go index bf9942fa..3b4e5d69 100644 --- a/internal/models/vendors/zhipu/vendor.go +++ b/internal/models/vendors/zhipu/vendor.go @@ -107,8 +107,19 @@ func init() { types.ModelTypeEmbedding, types.ModelTypeRerank, types.ModelTypeVLLM, + types.ModelTypeASR, }, Compat: catalog.VendorCompat{ + Transcriptions: catalog.TranscriptionsCompat{ + // https://docs.bigmodel.cn/api-reference/模型-api/语音转文本: + // POST {base}/audio/transcriptions, multipart file + model + // (glm-asr-2512), answering {text}. The file is ".wav / .mp3", + // at most 25 MB and 30 seconds — the duration cannot be checked + // here without decoding, so a longer file is the vendor's error. + MaxFileBytes: catalog.Ptr(25 << 20), + // No language field; prompt and hotwords are its only hints. + Formats: []string{"wav", "mp3"}, + }, Embeddings: catalog.EmbeddingsCompat{ // https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入: model, // input, dimensions — no encoding_format. embedding-2 is fixed at diff --git a/internal/utils/extraheaders.go b/internal/utils/extraheaders.go index b32d54f8..c6f644b8 100644 --- a/internal/utils/extraheaders.go +++ b/internal/utils/extraheaders.go @@ -44,44 +44,3 @@ func ApplyCustomHeaders(req *http.Request, headers map[string]string) { req.Header.Set(name, v) } } - -// CustomHeadersRoundTripper 是一个 http.RoundTripper 包装器, -// 会在每个 HTTP 请求发出前注入用户自定义的 header。 -// 用于无法直接拿到底层 *http.Request 的场景(如 go-openai SDK)。 -type CustomHeadersRoundTripper struct { - Headers map[string]string - Base http.RoundTripper -} - -// RoundTrip 实现 http.RoundTripper 接口。 -func (t *CustomHeadersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - base := t.Base - if base == nil { - base = http.DefaultTransport - } - if len(t.Headers) > 0 { - // 复制请求以避免修改调用方传入的对象。 - cloned := req.Clone(req.Context()) - ApplyCustomHeaders(cloned, t.Headers) - return base.RoundTrip(cloned) - } - return base.RoundTrip(req) -} - -// WrapHTTPClientWithHeaders 返回一个新的 *http.Client,在原有 client 基础上注入自定义 header。 -// 如果 headers 为空则直接返回原 client,避免不必要的开销。 -func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) *http.Client { - if len(headers) == 0 { - return client - } - if client == nil { - client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig()) - } - base := client.Transport - if base == nil { - base = http.DefaultTransport - } - wrapped := *client - wrapped.Transport = &CustomHeadersRoundTripper{Headers: headers, Base: base} - return &wrapped -} diff --git a/internal/utils/extraheaders_test.go b/internal/utils/extraheaders_test.go index 5f4ecf33..45177956 100644 --- a/internal/utils/extraheaders_test.go +++ b/internal/utils/extraheaders_test.go @@ -1,10 +1,7 @@ package utils import ( - "io" "net/http" - "net/http/httptest" - "strings" "testing" ) @@ -48,47 +45,3 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) { t.Fatalf("unexpected headers added: %+v", req.Header) } } - -func TestWrapHTTPClientWithHeaders(t *testing.T) { - t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost") - ResetSSRFWhitelistForTest() - t.Cleanup(ResetSSRFWhitelistForTest) - gotTrace := "" - gotAuth := "" - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotTrace = r.Header.Get("X-Trace-Id") - gotAuth = r.Header.Get("Authorization") - io.Copy(io.Discard, r.Body) - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() - - client := WrapHTTPClientWithHeaders(nil, map[string]string{ - "X-Trace-Id": "rt-1", - "Authorization": "Bearer should-not-override", - }) - - req, _ := http.NewRequest("POST", srv.URL, strings.NewReader("{}")) - req.Header.Set("Authorization", "Bearer kept") - - resp, err := client.Do(req) - if err != nil { - t.Fatalf("request failed: %v", err) - } - resp.Body.Close() - - if gotTrace != "rt-1" { - t.Fatalf("expected custom header injected, got %q", gotTrace) - } - if gotAuth != "Bearer kept" { - t.Fatalf("reserved header must not be overridden, got %q", gotAuth) - } -} - -func TestWrapHTTPClientWithHeaders_EmptyReturnsOriginal(t *testing.T) { - orig := &http.Client{} - wrapped := WrapHTTPClientWithHeaders(orig, nil) - if wrapped != orig { - t.Fatalf("expected original client returned when headers empty") - } -} diff --git a/website-docs/02-architecture/01-overview.md b/website-docs/02-architecture/01-overview.md index 6687441c..d49091db 100644 --- a/website-docs/02-architecture/01-overview.md +++ b/website-docs/02-architecture/01-overview.md @@ -58,7 +58,7 @@ WeKnora 采用"主服务 + 前端 + 文档解析微服务"的三进程核心架 | 配置 | `github.com/spf13/viper` + `config/config.yaml` + 环境变量 | — | | 可观测 | OpenTelemetry + Langfuse(`internal/tracing/langfuse`) | LLM 调用级 trace | | gRPC | `google.golang.org/grpc` v1.81.0 | 调用 docreader | -| LLM 接入 | `sashabaranov/go-openai`、Ollama、腾讯云 LKE 等 | 18+ 模型提供商(OpenAI 兼容 / Ollama / 云厂商 SDK) | +| LLM 接入 | 自研协议层 `internal/models/api`(OpenAI / Anthropic / Gemini / DashScope 等线格式各一个包)、Ollama、腾讯云 LKE SDK 等 | 27 家厂商由 `internal/models/vendors` 声明,见[模型管理](../03-features/06-models.md) | | 向量/检索 | pgvector、ES v7/v8、OpenSearch、Qdrant、Milvus、Weaviate、Doris、腾讯 VectorDB、sqlite-vec | 由 `RETRIEVE_DRIVER` 与 `vector_stores` 表动态装配 | | 知识图谱 | `neo4j-go-driver/v6` | 可选 | | 数据分析 | DuckDB(`duckdb-go/v2`)、`pg_query_go` SQL 校验 | Agent 数据分析工具 | diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.md index e0af171a..3d8ccf7e 100644 --- a/website-docs/03-features/06-models.md +++ b/website-docs/03-features/06-models.md @@ -56,7 +56,7 @@ const ( | `Embedding` | `embedding` | `internal/models/embedding` | `Embed` / `BatchEmbed`(含 `GetDimensions`) | 文本向量化,供向量检索索引与查询 | | `Rerank` | `rerank` | `internal/models/rerank` | `Rerank(query, documents)` 返回 `RankResult` | 检索结果精排 | | `VLLM` | `vllm` | `internal/models/vlm` | `Predict(imgBytes, prompt)` | 视觉语言模型(VLM),文档图片理解 / 多模态解析 | -| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本与分段时间戳 | 音频转写(自动语音识别) | +| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本;模型提供时附带分段时间戳(如 OpenAI `whisper-1`) | 音频转写(自动语音识别) | 前后端类型映射见 `internal/handler/model.go` 的 `modelTypeToFrontend()`(`KnowledgeQA -> chat` 等)。 @@ -176,7 +176,7 @@ builtin_models: | 层 | 位置 | 职责 | |----|------|------| -| 协议层 | `internal/models/api/` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`。各自持有请求/响应结构与解析,不含任何厂商名 | +| 协议层 | `internal/models/api/` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`;语音:`openaitranscriptions`。各自持有请求/响应结构与解析,不含任何厂商名 | | 厂商层 | `internal/models/vendors//` | 一个厂商一个目录:`vendor.go`(注册 `catalog.Vendor`)、`models.json`(模型目录)、`icon.svg`(品牌图标,`go:embed` 进二进制) | | 目录层 | `internal/models/catalog` | 合并厂商定义、模型条目、部署叠加与单行覆盖,`Resolve` 得出「这个模型到底怎么发请求」 | @@ -235,7 +235,7 @@ make model-catalog-check 前端不需要任何改动:厂商下拉、图标、额外字段、内置模型列表都由 `GET /api/v1/models/providers` 动态渲染。 -写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(chat、VLM、embedding、rerank 四类;ASR 还没进目录),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。 +写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(全部模型类型),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。 #### 厂商更新了模型怎么办 @@ -280,7 +280,21 @@ Embedding 行也有几处按厂商文档纠正的行为变化(逐厂商的出 7. **OpenAI 兼容回复里没有 `index` 时按顺序取**(和旧实现一致);有 `index` 就按它放回,重复或缺位报错。 8. **Jina 的 `task`、Gemini 的 `taskType`、OpenRouter 的 `input_type`、火山的 `instructions`、百炼原生接口的 `text_type` / `instruct` 都不发**。它们会改变文档侧向量,开启后同一个知识库里新旧向量不在同一空间;需要按行显式开启的设计另见 Tencent/WeKnora#1401。 -另外 Azure OpenAI 不再声明支持 ASR(ASR 客户端只会构造标准 OpenAI 客户端,根本无法带上 Azure 的 `api-key` 头和部署路径,这类行此前就调不通)。 +ASR 行的变化(逐厂商的出站表单由 `internal/models/asr/wire_test.go` 钉住): + +1. **不再一律发 `response_format=verbose_json`**。OpenAI 文档写明 gpt-4o-transcribe / gpt-4o-mini-transcribe「only supported format is json」,旧实现发给它们必然 400;GPUStack 的音频后端 vox-box 在 FunASR 模型(SenseVoice、Paraformer)上对 `verbose_json` 返回裸字符串,旧实现解不开。现在只有文档写明支持的模型声明它(OpenAI `whisper-1`),其余走默认 json,Langfuse 里的音频时长因此只在返回分段的模型上有。需要分段的自建行可以在 `spec.compat` 里写 `{"response_format": "verbose_json"}`。 +2. **按厂商文档的上限和格式在上传前拒绝**:文件大小(OpenAI / 智谱 / OpenRouter 25 MB、Requesty 32 MB、SiliconFlow / MiniMax 50 MB;阿里与小米按整段 `data:` URI 计 10 MB),以及文档给出封闭格式清单的厂商(智谱、小米只收 wav/mp3,OpenAI、Requesty、MiniMax 各有列表)。 +3. **回复里没有 `text` 字段即报错**,不再当成「未检测到语音」入库;静音音频返回的是空字符串 `text`,照常处理。 +4. ASR 行现在带上 provider 走目录解析,go-openai 依赖随之移除。provider 没有声明 ASR 的行:显式填了该厂商的直接报错;没填、靠 URL 识别到的,按 OpenAI 形状打到它的 URL(与旧实现一致),不走该厂商的 Endpoint 钩子。 +5. **支持 ASR 的厂商按文档补齐**。OpenAI 形状(multipart `file` + `model`):openai、siliconflow、gpustack、generic、智谱(`glm-asr-2512`,单文件 ≤30 秒)、MiniMax(`asr-1.0`,路径是 `/v1/speech_to_text`)、OpenRouter、Requesty、LiteLLM。经 chat completions 的 `input_audio`(base64 data URI,编码后 ≤10 MB):阿里云 `qwen3-asr-flash`、小米 `mimo-v2.5-asr`。 +6. **知识库的「音频语言提示」终于发出去了**(此前从未接线)。位置按厂商文档:OpenAI / Requesty / OpenRouter / GPUStack / generic 是 `language` 表单字段,MiniMax 是 `language` 请求头,阿里与小米是 `asr_options.language`;智谱、SiliconFlow、LiteLLM 文档没有这个参数,不发。填 `auto` 等同留空。 +7. **音频时长从回包里读**:MiniMax 的 `duration`、OpenAI 系与阿里的 `usage.seconds`,Langfuse 不再只靠 `verbose_json` 的分段。 + +阿里只有 `qwen3-asr-flash` 能直接带音频调用;其余 ASR 模型名由一条兜底条目拒绝并说明原因。 + +查过但没有接的:火山豆包语音(独立域名与密钥,当前文档的请求体只收音频 URL)、千帆(`vop_asr` 挂在应用实例下要 `app_id`;短语音接口 ≤60 秒且只收 pcm/wav/amr/m4a)、七牛(只收音频 URL)、Novita(GLM-ASR 自有接口,≤30 秒)、腾讯云 ASR(与混元不是同一套鉴权)、NVIDIA 托管 Riva(gRPC)、Gemini(没有专用转写接口,只能让对话模型听音频)、Azure(见下)。阿里 Paraformer / Fun-ASR / `*-filetrans` 是异步任务,同样要公网 URL。 + +Azure OpenAI 仍不声明 ASR:音频转写只出现在 v1 **preview** 参考里(`/openai/v1/audio/transcriptions?api-version=preview`),v1 GA 参考没有;未填 `api_version` 的行会被钩子发到不带版本的 v1 路径,能否调通没有核实。 ### 模型调用链