refactor(asr): move the wire format into protocol packages, and cover every vendor whose docs offer ASR (#3503)

main
lyingbug 2026-09-21 13:25:27 +08:00 committed by GitHub
parent 7d688cb8d3
commit 4a9aa06274
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
53 changed files with 1835 additions and 318 deletions

View File

@ -117,7 +117,7 @@ test:
.PHONY: model-catalog-check
model-catalog-check:
go test ./internal/models/parity/ ./internal/models/vendors/ ./internal/models/catalog/ ./internal/models/api/... \
./internal/models/rerank/ ./internal/models/embedding/
./internal/models/rerank/ ./internal/models/embedding/ ./internal/models/asr/
# Vendor catalog: report where our model metadata differs from models.dev.
# Development aid only — nothing is fetched at runtime and nothing is written

1
go.mod
View File

@ -51,7 +51,6 @@ require (
github.com/redis/go-redis/v9 v9.14.1
github.com/robfig/cron/v3 v3.0.1
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
github.com/sashabaranov/go-openai v1.41.2
github.com/sirupsen/logrus v1.9.4
github.com/slack-go/slack v0.23.1
github.com/spf13/viper v1.21.0

2
go.sum
View File

@ -2516,8 +2516,6 @@ github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=
github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM=
github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=

View File

@ -18,6 +18,7 @@ import (
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/searchutil"
@ -3699,7 +3700,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
transcriptionResult, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
// The knowledge base's language hint; the vendor declares where it
// goes, or that it takes none.
asrCtx := asr.WithLanguage(ctx, eff.ASRConfig.Language)
transcriptionResult, err := asrModel.Transcribe(asrCtx, convertResult.AudioData, knowledge.FileName)
if err != nil {
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
if isLastRetry {

View File

@ -759,8 +759,7 @@ func (h *ModelHandler) DeleteModel(c *gin.Context) {
// validateCatalogParameters rejects protocol / compat overrides the catalog
// cannot interpret (unknown extra_config.api, unknown compat keys, bad
// reasoning levels) so a typo fails at save time instead of at the first
// call. Chat, VLM, embedding and rerank rows are checked; see
// catalog.ValidateRow.
// call. Every model type is checked; see catalog.ValidateRow.
func validateCatalogParameters(name string, modelType types.ModelType, params *types.ModelParameters) error {
return catalog.ValidateRow(name, modelType, params)
}

View File

@ -100,24 +100,33 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b
if err != nil {
return nil, nil, fmt.Errorf("marshal request: %w", err)
}
if err := secutils.ValidateURLForSSRF(url); err != nil {
return nil, nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
req, err := e.newPost(ctx, url, data, "application/json")
if err != nil {
return nil, nil, fmt.Errorf("create request: %w", err)
return nil, nil, err
}
req.Header.Set("Content-Type", "application/json")
if stream {
req.Header.Set("Accept", "text/event-stream")
}
return req, data, nil
}
// newPost prepares an authenticated POST of an already-encoded body.
func (e Endpoint) newPost(ctx context.Context, url string, data []byte, contentType string) (*http.Request, error) {
if err := secutils.ValidateURLForSSRF(url); err != nil {
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", contentType)
if e.Auth != nil {
e.Auth(req, data)
}
// User headers are applied last but the helper skips reserved names so
// they cannot clobber auth or content negotiation.
secutils.ApplyCustomHeaders(req, e.Headers)
return req, data, nil
return req, nil
}
// Do sends the request and returns the response. Non-2xx responses are
@ -125,7 +134,7 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b
func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
resp, err := e.httpClient().Do(req)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
return nil, &TransportError{Op: "send request", Err: err}
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
@ -151,6 +160,11 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
// prompt is what an operator debugs, while a rerank body is a query plus
// every candidate chunk. The rerank layer logs a truncated line at Debug
// instead, so this would have duplicated it at Info.
return e.roundTrip(req, out)
}
// roundTrip sends a prepared request and decodes a 2xx reply into out.
func (e Endpoint) roundTrip(req *http.Request, out any) error {
resp, err := e.Do(req)
if err != nil {
return err
@ -158,7 +172,9 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
defer func() { _ = resp.Body.Close() }()
raw, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read response: %w", err)
// The connection broke before the reply arrived whole: nothing was
// received, so this is the network failing, not the vendor answering.
return &TransportError{Op: "read response", Err: err}
}
if out == nil {
return nil
@ -169,6 +185,18 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
return nil
}
// TransportError is a request that never got a whole answer: DNS, connect,
// TLS, a reset or a timeout, while sending or while reading the reply. It is
// the only failure worth sending again.
type TransportError struct {
// Op is the phase that failed: "send request" or "read response".
Op string
Err error
}
func (e *TransportError) Error() string { return e.Op + ": " + e.Err.Error() }
func (e *TransportError) Unwrap() error { return e.Err }
// HTTPError is a non-2xx vendor reply.
type HTTPError struct {
StatusCode int

View File

@ -0,0 +1,76 @@
package api
import (
"bytes"
"context"
"fmt"
"mime/multipart"
"net/textproto"
"path/filepath"
"strings"
)
// FormField is one text part of a multipart form. Order is kept.
type FormField struct {
Name string
Value string
}
// FormFile is the file part of a multipart form.
type FormFile struct {
Field string
FileName string
Data []byte
}
// EncodeMultipart writes fields and one file into a multipart/form-data body
// and returns it with its Content-Type. It is encoded once: those bytes are
// what a signing vendor hashes and what every retry sends again.
//
// The file part carries a filename and no Content-Type of its own. Servers
// identify the format from the extension then — OpenAI asks for "an
// extension-bearing filename", and vox-box (GPUStack's audio backend) falls
// back to guessing from it — whereas a guessed type that a server does not
// list is rejected outright.
func EncodeMultipart(fields []FormField, file FormFile) ([]byte, string, error) {
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
header := make(textproto.MIMEHeader)
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`,
file.Field, filepath.Base(file.FileName)))
part, err := w.CreatePart(header)
if err != nil {
return nil, "", fmt.Errorf("encode form file: %w", err)
}
if _, err := part.Write(file.Data); err != nil {
return nil, "", fmt.Errorf("encode form file: %w", err)
}
for _, f := range fields {
if err := w.WriteField(f.Name, f.Value); err != nil {
return nil, "", fmt.Errorf("encode form field %s: %w", f.Name, err)
}
}
if err := w.Close(); err != nil {
return nil, "", fmt.Errorf("encode form: %w", err)
}
return buf.Bytes(), w.FormDataContentType(), nil
}
// PostMultipartWithRetry sends an encoded multipart body, retrying transport
// failures only, and decodes the JSON reply into out. It has the guarantees
// of PostJSONWithRetry.
func (e Endpoint) PostMultipartWithRetry(
ctx context.Context, url string, body []byte, contentType string,
out any, policy RetryPolicy, label string,
) error {
if !strings.HasPrefix(contentType, "multipart/") {
return fmt.Errorf("%s: not a multipart content type: %q", label, contentType)
}
return withRetry(ctx, policy, label, func() error {
req, err := e.newPost(ctx, url, body, contentType)
if err != nil {
return err
}
return e.roundTrip(req, out)
})
}

View File

@ -0,0 +1,152 @@
// Package openaichataudio implements speech recognition served through the
// chat completions endpoint: one user message whose only content part is
// {"type": "input_audio", "input_audio": {"data": "data:<mime>;base64,..."}},
// answered by an assistant message whose content is the transcript.
//
// Only dedicated recognition models are addressed this way — Alibaba's
// qwen3-asr-flash and Xiaomi's mimo-v2.5-asr — so no instruction is sent: the
// model transcribes whatever it is given. A general audio-capable chat model
// would need a prompt and would answer about the audio rather than transcribe
// it, which is why this is its own protocol and not a chat request.
//
// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference
// https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition
package openaichataudio
import (
"context"
"encoding/base64"
"fmt"
"path/filepath"
"strings"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
)
// Config is everything the client needs, already resolved by the catalog.
type Config struct {
Endpoint api.Endpoint
Settings catalog.TranscriptionsSettings
Retry api.RetryPolicy
}
// Client talks chat-served speech recognition to one endpoint.
type Client struct {
cfg Config
}
// New builds a client.
func New(cfg Config) *Client { return &Client{cfg: cfg} }
const defaultPath = "/chat/completions"
func (c *Client) url() string {
if c.cfg.Endpoint.URL != "" {
return c.cfg.Endpoint.Resolve("")
}
path := defaultPath
if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
path = ""
}
return c.cfg.Endpoint.Resolve(path)
}
// audioTypes maps the file extension onto the media type of the data URI.
// The two references show audio/wav and audio/mpeg; the rest are the
// registered types for the containers Alibaba lists. A fixed table rather
// than mime.TypeByExtension, which reads the host's tables and would make the
// request depend on the machine it runs on.
var audioTypes = map[string]string{
".wav": "audio/wav",
".mp3": "audio/mpeg",
".m4a": "audio/mp4",
".mp4": "audio/mp4",
".aac": "audio/aac",
".flac": "audio/flac",
".ogg": "audio/ogg",
".opus": "audio/opus",
".webm": "audio/webm",
".amr": "audio/amr",
}
// DataURI is the input_audio payload for one file. Both references show an
// audio media type in the URI, so a file whose extension names no audio
// container is refused rather than labelled application/octet-stream.
func DataURI(audio []byte, fileName string) (string, error) {
mediaType, ok := audioTypes[strings.ToLower(filepath.Ext(fileName))]
if !ok {
return "", fmt.Errorf("cannot tell the audio format of %q from its extension", fileName)
}
return "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(audio), nil
}
// BuildRequestBody is the golden-test entry point.
func (c *Client) BuildRequestBody(req api.TranscriptionRequest) (map[string]any, error) {
uri, err := DataURI(req.Audio, req.FileName)
if err != nil {
return nil, err
}
// The ceiling is on the whole string as sent — "encoded string size must
// not exceed 10 MB" — so the data: prefix counts too.
if limit := c.cfg.Settings.MaxEncodedBytes; limit > 0 && len(uri) > limit {
return nil, fmt.Errorf(
"the audio is %.1f MB and %.1f MB once base64-encoded; this model accepts at most %.1f MB encoded",
float64(len(req.Audio))/(1<<20), float64(len(uri))/(1<<20), float64(limit)/(1<<20))
}
body := map[string]any{
"model": c.cfg.Endpoint.Model,
"messages": []any{map[string]any{
"role": "user",
"content": []any{map[string]any{
"type": "input_audio",
"input_audio": map[string]any{"data": uri},
}},
}},
}
if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageASROptions {
body["asr_options"] = map[string]any{"language": req.Language}
}
return body, nil
}
type response struct {
Choices []struct {
Message struct {
// Content is a pointer for the same reason as the multipart
// protocol's text: an absent transcript is an error, an empty one
// is silence.
Content *string `json:"content"`
} `json:"message"`
} `json:"choices"`
// Usage.Seconds is the audio length, which Alibaba reports.
Usage struct {
Seconds float64 `json:"seconds"`
} `json:"usage"`
Error *struct {
Message string `json:"message"`
} `json:"error"`
}
// Transcribe sends one audio file.
func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {
body, err := c.BuildRequestBody(req)
if err != nil {
return nil, err
}
var decoded response
err = c.cfg.Endpoint.PostJSONWithRetry(ctx, c.url(), body, &decoded, c.cfg.Retry, "transcription")
if err != nil {
return nil, err
}
if decoded.Error != nil && decoded.Error.Message != "" {
return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)
}
if len(decoded.Choices) == 0 || decoded.Choices[0].Message.Content == nil {
return nil, fmt.Errorf("transcription reply carries no message content")
}
return &api.Transcription{
Text: strings.TrimSpace(*decoded.Choices[0].Message.Content),
Duration: decoded.Usage.Seconds,
}, nil
}

View File

@ -0,0 +1,162 @@
package openaichataudio
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// qwenASRResponse is the non-streaming response example from Alibaba's
// Qwen-ASR reference, copied from the page:
// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference
const qwenASRResponse = `{
"choices": [{
"finish_reason": "stop",
"index": 0,
"message": {
"annotations": [{"emotion": "neutral", "language": "zh", "type": "audio_info"}],
"content": "欢迎使用阿里云。",
"role": "assistant"
}
}],
"created": 1767683986,
"id": "chatcmpl-487abe5f-d4f2-9363-a877-xxxxxxx",
"model": "qwen3-asr-flash",
"object": "chat.completion",
"usage": {"completion_tokens": 12, "prompt_tokens": 42, "total_tokens": 54}
}`
// withSeconds is a reply carrying the usage.seconds that the reference's
// schema documents ("音频时长(秒)") but its example omits.
const withSeconds = `{"choices":[{"message":{"role":"assistant","content":"x"}}],"usage":{"seconds":3}}`
func newClient(url, model string) *Client {
return newClientWith(url, model, catalog.TranscriptionsSettings{})
}
func newClientWith(url, model string, settings catalog.TranscriptionsSettings) *Client {
return New(Config{
Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},
Settings: settings,
})
}
// The request is the one both references show: a single user message whose
// only part is the audio as a data URI, no instruction, no stream flag.
func TestRequestBodyMatchesTheReferences(t *testing.T) {
body, err := newClient("https://example.invalid/v1", "qwen3-asr-flash").
BuildRequestBody(api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "clip.MP3"})
require.NoError(t, err)
assert.Equal(t, map[string]any{
"model": "qwen3-asr-flash",
"messages": []any{map[string]any{
"role": "user",
"content": []any{map[string]any{
"type": "input_audio",
"input_audio": map[string]any{"data": "data:audio/mpeg;base64,SUQz"},
}},
}},
}, body)
}
func TestDataURIMediaTypeFollowsTheExtension(t *testing.T) {
uri, err := DataURI([]byte("RIFF"), "a.wav")
require.NoError(t, err)
assert.Equal(t, "data:audio/wav;base64,UklGRg==", uri)
// No audio type can be named for it, and the references show one.
_, err = DataURI([]byte("x"), "noext")
assert.Error(t, err)
}
// Both references cap the string as sent, so the data: prefix counts. A file
// sized to exactly the encoded budget without the prefix goes over it.
func TestEncodedCeilingCountsTheWholeDataURI(t *testing.T) {
const limit = 1024
c := newClientWith("https://example.invalid", "m", catalog.TranscriptionsSettings{MaxEncodedBytes: limit})
audio := make([]byte, limit*3/4) // base64 of this is exactly limit bytes
_, err := c.BuildRequestBody(api.TranscriptionRequest{Audio: audio, FileName: "a.wav"})
require.Error(t, err)
prefix := len("data:audio/wav;base64,")
fits := make([]byte, (limit-prefix)/4*3)
_, err = c.BuildRequestBody(api.TranscriptionRequest{Audio: fits, FileName: "a.wav"})
require.NoError(t, err)
}
func TestLanguageGoesInASROptionsWhereDeclared(t *testing.T) {
req := api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav", Language: "zh"}
body, err := newClient("https://example.invalid", "m").BuildRequestBody(req)
require.NoError(t, err)
assert.NotContains(t, body, "asr_options", "undeclared, so not sent")
body, err = newClientWith("https://example.invalid", "m",
catalog.TranscriptionsSettings{LanguageParam: catalog.LanguageASROptions}).BuildRequestBody(req)
require.NoError(t, err)
assert.Equal(t, map[string]any{"language": "zh"}, body["asr_options"])
}
func TestDecodesTheDocumentedResponse(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
var path string
var sent map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&sent)
_, _ = w.Write([]byte(qwenASRResponse))
}))
defer server.Close()
out, err := newClient(server.URL+"/compatible-mode/v1", "qwen3-asr-flash").
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "a.wav"})
require.NoError(t, err)
assert.Equal(t, "/compatible-mode/v1/chat/completions", path)
assert.Equal(t, &api.Transcription{Text: "欢迎使用阿里云。"}, out)
}
// A reply with no assistant content is not silence.
func TestReplyWithoutContentIsAnError(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"choices":[]}`))
}))
defer server.Close()
_, err := newClient(server.URL, "m").
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
assert.Error(t, err)
}
func TestDurationFromUsageSeconds(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(withSeconds))
}))
defer server.Close()
out, err := newClient(server.URL, "m").
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.NoError(t, err)
assert.Equal(t, float64(3), out.Duration)
}
func TestEmptyContentIsSilence(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":""}}]}`))
}))
defer server.Close()
out, err := newClient(server.URL, "m").
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.NoError(t, err)
assert.Equal(t, "", out.Text)
}

View File

@ -0,0 +1,131 @@
// Package openaitranscriptions implements the speech-to-text shape OpenAI
// defined and the compatible servers copied: a multipart POST to
// {base}/audio/transcriptions carrying file and model, answering {text}, or
// {text, segments} when response_format is verbose_json.
//
// response_format is the one optional field that matters, and it differs by
// model: gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json,
// whisper-1 also serves verbose_json, SiliconFlow documents no such field at
// all. json is the default everywhere it is documented, so nothing is sent
// unless catalog.TranscriptionsSettings names a format.
//
// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create
package openaitranscriptions
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
)
// Config is everything the client needs, already resolved by the catalog.
type Config struct {
Endpoint api.Endpoint
Settings catalog.TranscriptionsSettings
Retry api.RetryPolicy
}
// Client talks the OpenAI transcription shape to one endpoint.
type Client struct {
cfg Config
}
// New builds a client.
func New(cfg Config) *Client { return &Client{cfg: cfg} }
func (c *Client) url() string {
if c.cfg.Endpoint.URL != "" {
return c.cfg.Endpoint.Resolve("")
}
path := c.cfg.Settings.Path
if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
path = ""
}
return c.cfg.Endpoint.Resolve(path)
}
// FormFields is the golden-test entry point: the text parts of the form, in
// the order they are written.
func (c *Client) FormFields(language string) []api.FormField {
fields := []api.FormField{{Name: "model", Value: c.cfg.Endpoint.Model}}
if format := c.cfg.Settings.ResponseFormat; format != "" {
fields = append(fields, api.FormField{Name: "response_format", Value: format})
}
if language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageForm {
fields = append(fields, api.FormField{Name: "language", Value: language})
}
return fields
}
type response struct {
// Text is a pointer because its absence is an error, not silence. A
// transcript of silent audio is a present, empty string; a reply without
// the field is some other document. vox-box (GPUStack's audio backend)
// returns its HTTPException from the route instead of raising it, so a
// failed transcription can arrive as a 2xx whose body is the exception.
Text *string `json:"text"`
Segments []struct {
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
} `json:"segments"`
// Duration is top-level in OpenAI's verbose_json and in MiniMax's
// default json; usage.seconds is where OpenAI, OpenRouter and Requesty
// report it for models billed by duration.
Duration float64 `json:"duration"`
Usage struct {
Seconds float64 `json:"seconds"`
} `json:"usage"`
Detail any `json:"detail"`
Error *struct {
Message string `json:"message"`
} `json:"error"`
}
// Transcribe sends one audio file.
func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {
body, contentType, err := api.EncodeMultipart(c.FormFields(req.Language), api.FormFile{
Field: "file", FileName: req.FileName, Data: req.Audio,
})
if err != nil {
return nil, err
}
endpoint := c.cfg.Endpoint
if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageHeader {
headers := make(map[string]string, len(endpoint.Headers)+1)
for k, v := range endpoint.Headers {
headers[k] = v
}
headers["language"] = req.Language
endpoint.Headers = headers
}
var decoded response
err = endpoint.PostMultipartWithRetry(
ctx, c.url(), body, contentType, &decoded, c.cfg.Retry, "transcription",
)
if err != nil {
return nil, err
}
if decoded.Error != nil && decoded.Error.Message != "" {
return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)
}
if decoded.Text == nil {
if decoded.Detail != nil {
return nil, fmt.Errorf("transcription reply carries no text: %v", decoded.Detail)
}
return nil, fmt.Errorf("transcription reply carries no text")
}
out := &api.Transcription{Text: strings.TrimSpace(*decoded.Text), Duration: decoded.Duration}
if out.Duration == 0 {
out.Duration = decoded.Usage.Seconds
}
for _, s := range decoded.Segments {
out.Segments = append(out.Segments, api.TranscriptionSegment{
Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text),
})
}
return out, nil
}

View File

@ -0,0 +1,219 @@
package openaitranscriptions
import (
"context"
"io"
"mime"
"mime/multipart"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// jsonResponse is the json example from the reference, copied from the page:
// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create
const jsonResponse = `{
"text": "text",
"languages": [{"code": "code"}],
"logprobs": [{"token": "token", "bytes": [0], "logprob": 0}],
"usage": {
"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "type": "tokens",
"input_token_details": {"audio_tokens": 0, "text_tokens": 0}
}
}`
// verboseResponse is the "Segment timestamps" example from the same page,
// cut to its first segment where the page elides the rest with "...".
const verboseResponse = `{
"task": "transcribe",
"language": "english",
"duration": 8.470000267028809,
"text": "The beach was a popular spot on a hot summer day. ` +
`People were swimming in the ocean, building sandcastles, and playing beach volleyball.",
"segments": [
{
"id": 0,
"seek": 0,
"start": 0.0,
"end": 3.319999933242798,
"text": " The beach was a popular spot on a hot summer day.",
"tokens": [50364, 440, 7534, 390, 257, 3743, 4008, 322, 257, 2368, 4266, 786, 13, 50530],
"temperature": 0.0,
"avg_logprob": -0.2860786020755768,
"compression_ratio": 1.2363636493682861,
"no_speech_prob": 0.00985979475080967
}
],
"usage": {"type": "duration", "seconds": 9}
}`
type captured struct {
path string
auth string
language string
fields map[string]string
fileName string
fileType string
file []byte
}
func serve(t *testing.T, status int, reply string) (*httptest.Server, *captured) {
t.Helper()
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
got := &captured{fields: map[string]string{}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got.path = r.URL.Path
got.auth = r.Header.Get("Authorization")
got.language = r.Header.Get("language")
_, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
require.NoError(t, err)
reader := multipart.NewReader(r.Body, params["boundary"])
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
require.NoError(t, err)
data, _ := io.ReadAll(part)
if part.FormName() == "file" {
got.fileName, got.fileType, got.file = part.FileName(), part.Header.Get("Content-Type"), data
continue
}
got.fields[part.FormName()] = string(data)
}
w.WriteHeader(status)
_, _ = w.Write([]byte(reply))
}))
t.Cleanup(server.Close)
return server, got
}
func newClient(url, model string, settings catalog.TranscriptionsSettings) *Client {
if settings.Path == "" {
settings.Path = catalog.DefaultTranscriptions().Path
}
return New(Config{
Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},
Settings: settings,
})
}
// The baseline form is file and model. gpt-4o-transcribe accepts only json,
// which is the default, so no response_format is sent.
func TestBaselineFormIsFileAndModel(t *testing.T) {
server, got := serve(t, http.StatusOK, jsonResponse)
out, err := newClient(server.URL+"/v1", "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "meeting.wav"})
require.NoError(t, err)
assert.Equal(t, "/v1/audio/transcriptions", got.path)
assert.Equal(t, "Bearer k", got.auth)
assert.Equal(t, map[string]string{"model": "gpt-4o-transcribe"}, got.fields)
assert.Equal(t, "meeting.wav", got.fileName, "the extension is how the server identifies the format")
assert.Empty(t, got.fileType, "a guessed type a server does not list is rejected outright")
assert.Equal(t, []byte("RIFF"), got.file)
assert.Equal(t, &api.Transcription{Text: "text"}, out)
}
// The language hint goes where the vendor documents it: a form field on
// OpenAI's shape, a request header on MiniMax's, nowhere otherwise.
func TestLanguageGoesWhereTheVendorDeclares(t *testing.T) {
for _, tc := range []struct {
param string
wantField string
wantHeader string
}{
{param: "", wantField: "", wantHeader: ""},
{param: catalog.LanguageForm, wantField: "zh", wantHeader: ""},
{param: catalog.LanguageHeader, wantField: "", wantHeader: "zh"},
} {
t.Run("param="+tc.param, func(t *testing.T) {
server, got := serve(t, http.StatusOK, jsonResponse)
_, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{LanguageParam: tc.param}).
Transcribe(context.Background(), api.TranscriptionRequest{
Audio: []byte("x"), FileName: "a.wav", Language: "zh",
})
require.NoError(t, err)
assert.Equal(t, tc.wantField, got.fields["language"])
assert.Equal(t, tc.wantHeader, got.language)
})
}
}
// The audio length is read from wherever the reply states it, so usage can
// be reported without asking for segments.
func TestDurationIsReadFromTheReply(t *testing.T) {
for name, reply := range map[string]string{
// MiniMax's default json, from its reference.
"top-level duration": `{"text": "x", "duration": 26.325, "trace_id": "021785229015510a2c883cf675b9804d"}`,
// OpenRouter's reference example.
"usage.seconds": `{"text": "x", "usage": {"cost": 0.000508, "input_tokens": 83, "output_tokens": 30, ` +
`"seconds": 9.2, "total_tokens": 113}}`,
} {
t.Run(name, func(t *testing.T) {
server, _ := serve(t, http.StatusOK, reply)
out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.NoError(t, err)
assert.NotZero(t, out.Duration)
})
}
}
func TestVerboseJSONCarriesSegments(t *testing.T) {
server, got := serve(t, http.StatusOK, verboseResponse)
settings := catalog.TranscriptionsSettings{ResponseFormat: "verbose_json"}
out, err := newClient(server.URL+"/v1", "whisper-1", settings).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "a.mp3"})
require.NoError(t, err)
assert.Equal(t, "verbose_json", got.fields["response_format"])
require.Len(t, out.Segments, 1)
assert.Equal(t, api.TranscriptionSegment{
Start: 0, End: 3.319999933242798, Text: "The beach was a popular spot on a hot summer day.",
}, out.Segments[0])
}
// Silence is an empty transcript, and that is a valid answer.
func TestEmptyTextIsSilence(t *testing.T) {
server, _ := serve(t, http.StatusOK, `{"text": ""}`)
out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.NoError(t, err)
assert.Equal(t, "", out.Text)
}
// A reply without text is not silence. vox-box returns its HTTPException
// from the route rather than raising it, so a failure can arrive as a 2xx
// body shaped like this; reading it as an empty transcript would store "no
// speech detected" for a file that was never transcribed.
func TestReplyWithoutTextIsAnError(t *testing.T) {
server, _ := serve(t, http.StatusOK,
`{"status_code": 500, "detail": "Failed to transcribe audio, boom", "headers": null}`)
_, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.Error(t, err)
assert.Contains(t, err.Error(), "Failed to transcribe audio")
}
func TestSurfacesTheVendorErrorBody(t *testing.T) {
// A stand-in rejection in OpenAI's error envelope.
server, _ := serve(t, http.StatusBadRequest, `{"error":{"message":"stand-in rejection"}}`)
_, err := newClient(server.URL, "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.Error(t, err)
assert.Contains(t, err.Error(), "stand-in rejection")
}
func TestDoesNotDoubleThePath(t *testing.T) {
server, got := serve(t, http.StatusOK, jsonResponse)
_, err := newClient(server.URL+"/v1/audio/transcriptions", "m", catalog.TranscriptionsSettings{}).
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
require.NoError(t, err)
assert.Equal(t, "/v1/audio/transcriptions", got.path)
}

View File

@ -49,12 +49,23 @@ func (p RetryPolicy) delay(attempt int) time.Duration {
// Here the last error is captured in the enclosing scope on purpose and the
// function cannot fall through without it.
//
// Only transport failures are retried. A non-2xx reply is the vendor
// answering, and repeating a rejected request neither fixes it nor tells the
// operator anything new.
// Only transport failures are retried, including a reply cut off while it
// was being read. A non-2xx reply is the vendor answering, and repeating a
// rejected request neither fixes it nor tells the operator anything new; a
// reply that arrived whole but does not decode is the same answer every time,
// and sending it again would bill the request twice.
func (e Endpoint) PostJSONWithRetry(
ctx context.Context, url string, body, out any, policy RetryPolicy, label string,
) error {
return withRetry(ctx, policy, label, func() error {
return e.PostJSON(ctx, url, body, out)
})
}
// withRetry runs send until it succeeds, fails with anything other than a
// TransportError, or the budget runs out. The last error lives in the
// enclosing scope and the loop cannot fall through without it.
func withRetry(ctx context.Context, policy RetryPolicy, label string, send func() error) error {
var lastErr error
for attempt := 0; attempt <= policy.MaxRetries; attempt++ {
if attempt > 0 {
@ -67,12 +78,12 @@ func (e Endpoint) PostJSONWithRetry(
return ctx.Err()
}
}
err := e.PostJSON(ctx, url, body, out)
err := send()
if err == nil {
return nil
}
var httpErr *HTTPError
if errors.As(err, &httpErr) {
var transportErr *TransportError
if !errors.As(err, &transportErr) {
return err
}
lastErr = err

View File

@ -78,6 +78,25 @@ func TestPostJSONWithRetryDoesNotRetryAVendorRejection(t *testing.T) {
assert.Equal(t, 1, attempts, "a rejection must not be repeated")
}
// A reply that arrived but does not decode is not a transport failure: the
// vendor did the work, and sending the request again bills it again.
func TestPostJSONWithRetryDoesNotRetryAnUndecodableReply(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
attempts++
_, _ = w.Write([]byte(`not json`))
}))
defer server.Close()
var out map[string]any
err := Endpoint{}.PostJSONWithRetry(
context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",
)
require.Error(t, err)
assert.Equal(t, 1, attempts)
}
func TestPostJSONWithRetryStopsWhenTheContextIsDone(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
@ -101,3 +120,54 @@ func TestRetryPolicyBackoffDoublesAndIsCapped(t *testing.T) {
assert.Equal(t, 4*time.Second, p.delay(4), "capped")
assert.Zero(t, RetryPolicy{MaxRetries: 2}.delay(1), "no base delay means no waiting")
}
// A reply cut off mid-body never arrived, so it is sent again.
func TestPostJSONWithRetryRetriesAReplyCutOffWhileReading(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
attempts++
if attempts == 1 {
// Promise more than is sent, then hang up.
w.Header().Set("Content-Length", "100")
_, _ = w.Write([]byte(`{"a":`))
w.(http.Flusher).Flush() // the status and half a body are on the wire
hj, ok := w.(http.Hijacker)
require.True(t, ok)
conn, _, err := hj.Hijack()
require.NoError(t, err)
_ = conn.Close()
return
}
_, _ = w.Write([]byte(`{"a":1}`))
}))
defer server.Close()
var out map[string]any
err := Endpoint{}.PostJSONWithRetry(
context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",
)
require.NoError(t, err)
assert.Equal(t, 2, attempts)
assert.Equal(t, float64(1), out["a"])
}
// The phase that failed is named, so the log says whether the request or the
// reply was lost.
func TestReadFailureIsReportedAsSuch(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "100")
_, _ = w.Write([]byte(`{"a":`))
w.(http.Flusher).Flush()
conn, _, err := w.(http.Hijacker).Hijack()
require.NoError(t, err)
_ = conn.Close()
}))
defer server.Close()
err := Endpoint{}.PostJSON(context.Background(), server.URL, map[string]any{}, &map[string]any{})
var transportErr *TransportError
require.ErrorAs(t, err, &transportErr)
assert.Equal(t, "read response", transportErr.Op)
}

View File

@ -0,0 +1,56 @@
package api
import "context"
// TranscriptionAPI names a speech-to-text wire protocol. Like RerankAPI and
// EmbeddingAPI it is its own type, so no other modality's value validates.
type TranscriptionAPI string
// The transcription protocols WeKnora speaks.
const (
// TranscriptionOpenAI is POST {base}/audio/transcriptions as a multipart
// form carrying file and model, answering {text} — or {text, segments}
// when verbose_json is asked for.
TranscriptionOpenAI TranscriptionAPI = "openai-transcriptions"
// TranscriptionChatAudio is a dedicated speech-recognition model behind
// POST {base}/chat/completions: the audio goes in as a base64 data URI in
// an input_audio content part and the transcript comes back as the
// assistant message. Alibaba's qwen3-asr-flash and Xiaomi's mimo-v2.5-asr
// take no instruction; the model only transcribes.
TranscriptionChatAudio TranscriptionAPI = "openai-chat-audio"
)
// Known reports whether the value names a protocol this build implements.
func (a TranscriptionAPI) Known() bool {
return a == TranscriptionOpenAI || a == TranscriptionChatAudio
}
// TranscriptionSegment is one timed stretch of a transcript.
type TranscriptionSegment struct {
Start float64
End float64
Text string
}
// Transcription is what a protocol client returns.
type Transcription struct {
Text string
Segments []TranscriptionSegment
// Duration is the audio length in seconds when the reply states it —
// OpenAI-shaped duration or usage.seconds — and 0 otherwise.
Duration float64
}
// TranscriptionRequest is one file to transcribe.
type TranscriptionRequest struct {
Audio []byte
FileName string
// Language is the operator's hint, empty for auto-detection. Where it is
// sent, and whether at all, is the vendor's declaration.
Language string
}
// Transcriber turns one audio file into text.
type Transcriber interface {
Transcribe(ctx context.Context, req TranscriptionRequest) (*Transcription, error)
}

View File

@ -2,6 +2,7 @@ package asr
import (
"context"
"strings"
"github.com/Tencent/WeKnora/internal/types"
)
@ -17,6 +18,27 @@ type Segment struct {
type TranscriptionResult struct {
Text string `json:"text"`
Segments []Segment `json:"segments,omitempty"`
// Duration is the audio length in seconds when the vendor reports it,
// which several do without segments; 0 when unknown.
Duration float64 `json:"duration,omitempty"`
}
type languageKey struct{}
// WithLanguage attaches the operator's language hint (for example "zh") to
// a transcription. Empty and "auto" mean auto-detection, which is every
// vendor's default, and send nothing.
func WithLanguage(ctx context.Context, language string) context.Context {
return context.WithValue(ctx, languageKey{}, language)
}
func languageFrom(ctx context.Context) string {
language, _ := ctx.Value(languageKey{}).(string)
language = strings.TrimSpace(language)
if strings.EqualFold(language, "auto") {
return ""
}
return language
}
// ASR defines the interface for Automatic Speech Recognition model operations.
@ -35,7 +57,10 @@ type Config struct {
ModelName string
APIKey string
ModelID string
Language string // optional: specify language for transcription
// Provider is the vendor id stored on the row; empty detects it from
// BaseURL.
Provider string
ExtraConfig map[string]string
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
CustomHeaders map[string]string
}
@ -53,13 +78,14 @@ func ConfigFromModel(m *types.Model) *Config {
BaseURL: m.Parameters.BaseURL,
ModelName: m.Name,
Source: m.Source,
Provider: m.Parameters.Provider,
ExtraConfig: m.Parameters.ExtraConfig,
CustomHeaders: m.Parameters.CustomHeaders,
}
}
// NewASR creates an ASR instance based on the provided configuration.
// All ASR vendors use the OpenAI-compatible /v1/audio/transcriptions API.
func NewASR(config *Config) (ASR, error) {
a, err := NewOpenAIASR(config)
a, err := newASR(config)
return wrapASRLangfuse(a, err)
}

View File

@ -14,6 +14,8 @@ func TestConfigFromModel(t *testing.T) {
Parameters: types.ModelParameters{
BaseURL: "https://api.example.com/v1",
APIKey: "sk",
Provider: "siliconflow",
ExtraConfig: map[string]string{"remote_model_name": "r"},
CustomHeaders: map[string]string{"X": "y"},
},
}
@ -24,6 +26,11 @@ func TestConfigFromModel(t *testing.T) {
if cfg.BaseURL != "https://api.example.com/v1" || cfg.APIKey != "sk" {
t.Errorf("connection fields mismatch: %+v", cfg)
}
// The vendor decides the wire format; without it every row resolves to
// the generic runtime.
if cfg.Provider != "siliconflow" || cfg.ExtraConfig["remote_model_name"] != "r" {
t.Errorf("vendor fields not propagated: %+v", cfg)
}
if cfg.CustomHeaders["X"] != "y" {
t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)
}

View File

@ -8,7 +8,8 @@ import (
// langfuseASR wraps an ASR implementation and reports each Transcribe call
// as a Langfuse generation observation. Audio bytes are not uploaded — we
// record file name, audio size and duration (derived from segment end times)
// record file name, audio size and duration (as the vendor reports it, or
// from segment end times)
// so usage can be billed by second/minute, which is how most ASR providers
// price their services.
type langfuseASR struct {
@ -44,8 +45,13 @@ func (l *langfuseASR) Transcribe(ctx context.Context, audioBytes []byte, fileNam
if result != nil {
output["text"] = result.Text
output["segment_count"] = len(result.Segments)
if n := len(result.Segments); n > 0 {
// The vendor's own figure when it gives one; otherwise the end of
// the last segment, which only verbose replies carry.
duration = result.Duration
if n := len(result.Segments); duration == 0 && n > 0 {
duration = result.Segments[n-1].End
}
if duration > 0 {
output["duration_seconds"] = duration
}
}

View File

@ -1,150 +0,0 @@
package asr
import (
"bytes"
"context"
"encoding/json"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
openai "github.com/sashabaranov/go-openai"
)
const (
asrDefaultTimeout = 300 * time.Second // audio transcription can be slow
)
// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API.
type OpenAIASR struct {
modelName string
modelID string
client *openai.Client
baseURL string
language string
}
// NewOpenAIASR creates an OpenAI-compatible ASR instance.
func NewOpenAIASR(config *Config) (*OpenAIASR, error) {
if err := validateASRBaseURL(config.BaseURL); err != nil {
return nil, err
}
apiCfg := openai.DefaultConfig(config.APIKey)
if config.BaseURL != "" {
apiCfg.BaseURL = config.BaseURL
}
httpClient := newASRHTTPClient(asrDefaultTimeout)
// 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers)
if len(config.CustomHeaders) > 0 {
apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders)
} else {
apiCfg.HTTPClient = httpClient
}
return &OpenAIASR{
modelName: config.ModelName,
modelID: config.ModelID,
client: openai.NewClientWithConfig(apiCfg),
baseURL: config.BaseURL,
language: config.Language,
}, nil
}
// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.
func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error) {
if len(audioBytes) == 0 {
return nil, fmt.Errorf("audio bytes are empty")
}
// Ensure fileName has an extension for proper MIME type detection
if fileName == "" {
fileName = "audio.mp3"
}
logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s",
s.modelName, s.baseURL, len(audioBytes), fileName)
req := openai.AudioRequest{
Model: s.modelName,
FilePath: fileName,
Reader: bytes.NewReader(audioBytes),
Format: openai.AudioResponseFormatVerboseJSON,
}
if s.language != "" {
req.Language = s.language
}
resp, err := s.client.CreateTranscription(ctx, req)
if err != nil {
return nil, fmt.Errorf("ASR transcription request failed: %w", err)
}
jsonData, err := json.Marshal(resp)
if err != nil {
return nil, fmt.Errorf("marshal transcription response: %w", err)
}
logger.Debugf(ctx, "[ASR] Transcription response: %s", string(jsonData))
text := strings.TrimSpace(resp.Text)
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))
var segments []Segment
for _, s := range resp.Segments {
segments = append(segments, Segment{
Start: s.Start,
End: s.End,
Text: strings.TrimSpace(s.Text),
})
}
return &TranscriptionResult{
Text: text,
Segments: segments,
}, nil
}
func (s *OpenAIASR) GetModelName() string { return s.modelName }
func (s *OpenAIASR) GetModelID() string { return s.modelID }
// DetectAudioFormat returns a file extension hint for the given audio bytes.
func DetectAudioFormat(data []byte, fileName string) string {
if fileName != "" {
ext := strings.ToLower(filepath.Ext(fileName))
if ext != "" {
return ext
}
}
// Try to detect from magic bytes
if len(data) >= 4 {
// MP3: starts with ID3 or 0xFF 0xFB
if data[0] == 'I' && data[1] == 'D' && data[2] == '3' {
return ".mp3"
}
if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {
return ".mp3"
}
// FLAC: starts with "fLaC"
if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' {
return ".flac"
}
// OGG: starts with "OggS"
if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' {
return ".ogg"
}
// WAV: starts with "RIFF"
if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' {
return ".wav"
}
}
// M4A: check for ftyp box
if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' {
return ".m4a"
}
return ".mp3" // default
}

View File

@ -0,0 +1,183 @@
package asr
import (
"context"
"fmt"
"path/filepath"
"slices"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/api/openaichataudio"
"github.com/Tencent/WeKnora/internal/models/api/openaitranscriptions"
"github.com/Tencent/WeKnora/internal/models/catalog"
// catalog.Resolve answers from the vendor catalog, which is empty until
// the vendor packages have run their init.
_ "github.com/Tencent/WeKnora/internal/models/vendors"
"github.com/Tencent/WeKnora/internal/types"
)
// retryPolicy is the transport-error retry budget, which is none. The
// knowledge pipeline retries a failed transcription as a whole task, and a
// client-level retry on top would turn one hung 300-second upload into
// several.
var retryPolicy = func() api.RetryPolicy { return api.RetryPolicy{} }
// newASR resolves the catalog and returns the protocol client for the
// configured model. It mirrors rerank.newReranker and
// embedding.newRemoteEmbedder: the vendor's facts decide the protocol, the
// URL and the credential, and this function knows no vendor names.
func newASR(config *Config) (ASR, error) {
if config == nil {
return nil, fmt.Errorf("asr config is nil")
}
if strings.TrimSpace(config.ModelName) == "" {
return nil, fmt.Errorf("model name is required")
}
resolved, err := catalog.Resolve(catalog.Ref{
Provider: config.Provider,
Model: config.ModelName,
BaseURL: config.BaseURL,
ModelType: types.ModelTypeASR,
Extra: config.ExtraConfig,
})
if err != nil {
return nil, err
}
// A vendor that does not declare speech recognition has not been checked
// for it, and its Endpoint hook may still compute a path — Azure's does.
// A row that named the vendor is refused; a row that named none and was
// matched by its URL is what the pre-catalog client served: an
// OpenAI-compatible endpoint at that URL.
if !resolved.Vendor.SupportsType(types.ModelTypeASR) {
if strings.TrimSpace(config.Provider) != "" {
return nil, fmt.Errorf("%s does not offer speech recognition in this build", resolved.Vendor.Name)
}
resolved, err = catalog.Resolve(catalog.Ref{
Provider: catalog.GenericID,
Model: config.ModelName,
BaseURL: config.BaseURL,
ModelType: types.ModelTypeASR,
Extra: config.ExtraConfig,
})
if err != nil {
return nil, err
}
}
if err := validateASRBaseURL(resolved.BaseURL); err != nil {
return nil, err
}
vendor := resolved.Vendor
creds := catalog.Credentials{APIKey: config.APIKey}
if creds.APIKey == "" {
creds.APIKey = vendor.DefaultAPIKey
}
// Vendor headers first so a user header cannot silently replace a vendor
// beta flag, matching chat.NewRemoteChat.
headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders))
for k, v := range vendor.Headers {
headers[k] = v
}
for k, v := range config.CustomHeaders {
headers[k] = v
}
settings := resolved.Transcriptions
endpoint := api.Endpoint{
BaseURL: resolved.BaseURL,
Model: resolved.RemoteModel,
ModelID: config.ModelID,
Auth: vendor.AuthFunc(vendor.API, creds),
Headers: headers,
Client: newASRHTTPClient(time.Duration(settings.RequestTimeout) * time.Second),
}
if vendor.Endpoint != nil {
if url, query := vendor.Endpoint(catalog.EndpointRequest{
BaseURL: resolved.BaseURL,
Model: resolved.RemoteModel,
ModelType: types.ModelTypeASR,
API: vendor.API,
Extra: config.ExtraConfig,
}); url != "" {
if err := validateASRBaseURL(url); err != nil {
return nil, err
}
endpoint.URL, endpoint.Query = url, query
}
}
var client api.Transcriber
switch resolved.TranscriptionAPI {
case api.TranscriptionOpenAI:
client = openaitranscriptions.New(openaitranscriptions.Config{
Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),
})
case api.TranscriptionChatAudio:
client = openaichataudio.New(openaichataudio.Config{
Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),
})
default:
return nil, fmt.Errorf("unsupported transcription api %q for provider %s",
resolved.TranscriptionAPI, vendor.ID)
}
return &protocolASR{
inner: client,
settings: settings,
vendor: vendor.Name,
endpoint: resolved.BaseURL,
modelName: config.ModelName,
modelID: config.ModelID,
}, nil
}
// protocolASR adapts a protocol client to the ASR interface and refuses an
// upload the vendor has documented it will not take, before sending it.
type protocolASR struct {
inner api.Transcriber
settings catalog.TranscriptionsSettings
vendor string
endpoint string
modelName string
modelID string
}
func (a *protocolASR) GetModelName() string { return a.modelName }
func (a *protocolASR) GetModelID() string { return a.modelID }
func (a *protocolASR) Transcribe(ctx context.Context, audio []byte, fileName string) (*TranscriptionResult, error) {
if len(audio) == 0 {
return nil, fmt.Errorf("audio bytes are empty")
}
if limit := a.settings.MaxFileBytes; limit > 0 && len(audio) > limit {
return nil, fmt.Errorf("%s transcription: the audio is %.1f MB; %s accepts at most %d MB per file",
a.modelName, float64(len(audio))/(1<<20), a.vendor, limit>>20)
}
// The server identifies the format from the extension.
if fileName == "" {
fileName = "audio.mp3"
}
if formats := a.settings.Formats; len(formats) > 0 {
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(fileName)), ".")
if !slices.Contains(formats, ext) {
return nil, fmt.Errorf("%s transcription: %s accepts %s audio, not %q",
a.modelName, a.vendor, strings.Join(formats, "/"), fileName)
}
}
logger.Infof(ctx, "[ASR] transcribing model=%s endpoint=%s size=%d file=%s",
a.modelName, a.endpoint, len(audio), fileName)
out, err := a.inner.Transcribe(ctx, api.TranscriptionRequest{
Audio: audio, FileName: fileName, Language: languageFrom(ctx),
})
if err != nil {
return nil, fmt.Errorf("ASR transcription request failed: %w", err)
}
result := &TranscriptionResult{Text: out.Text, Duration: out.Duration}
for _, s := range out.Segments {
result.Segments = append(result.Segments, Segment{Start: s.Start, End: s.End, Text: s.Text})
}
logger.Infof(ctx, "[ASR] transcription completed, text length=%d", len(result.Text))
return result, nil
}

View File

@ -1,26 +0,0 @@
package asr
import (
"testing"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
func withASRSSRFWhitelist(t *testing.T, raw string) {
t.Helper()
t.Setenv("SSRF_WHITELIST", raw)
secutils.ResetSSRFWhitelistForTest()
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
}
func TestOpenAIASRRejectsInternalBaseURL(t *testing.T) {
withASRSSRFWhitelist(t, "")
_, err := NewOpenAIASR(&Config{
BaseURL: "http://169.254.169.254/latest/meta-data/",
ModelName: "asr-test",
})
if err == nil {
t.Fatalf("NewOpenAIASR returned nil error for blocked internal BaseURL")
}
}

View File

@ -0,0 +1,356 @@
package asr
import (
"context"
"encoding/json"
"io"
"mime"
"mime/multipart"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func allowLoopback(t *testing.T) {
t.Helper()
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
secutils.ResetSSRFWhitelistForTest()
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
}
type form struct {
path string
auth string
fields map[string]string
fileName string
language string // the language request header
// body is set instead of fields when the request was JSON: the
// chat-served recognisers take the audio as a data URI.
body map[string]any
}
// upstream records the form it receives and answers the documented json
// shape, adding a segment when verbose_json was asked for.
func upstream(t *testing.T) (string, *form, *atomic.Int32) {
t.Helper()
allowLoopback(t)
got := &form{fields: map[string]string{}}
calls := &atomic.Int32{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
got.path, got.auth, got.language = r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("language")
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
require.NoError(t, err)
if mediaType == "application/json" {
require.NoError(t, json.NewDecoder(r.Body).Decode(&got.body))
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":" hello "}}]}`))
return
}
reader := multipart.NewReader(r.Body, params["boundary"])
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
require.NoError(t, err)
data, _ := io.ReadAll(part)
if part.FormName() == "file" {
got.fileName = part.FileName()
continue
}
got.fields[part.FormName()] = string(data)
}
if got.fields["response_format"] == "verbose_json" {
_, _ = w.Write([]byte(`{"text":" hello ","segments":[{"start":0,"end":1.5,"text":" hello "}]}`))
return
}
_, _ = w.Write([]byte(`{"text":" hello "}`))
}))
t.Cleanup(server.Close)
return server.URL, got, calls
}
// TestTranscriptionWireFormatPerVendor pins the form each ASR vendor is sent.
// response_format appears only where the vendor documents it for that model:
// the pre-catalog client sent verbose_json to every one of them, which
// gpt-4o-transcribe rejects ("the only supported format is json") and which
// vox-box's FunASR backend answers with an undecodable bare string.
func TestTranscriptionWireFormatPerVendor(t *testing.T) {
chatAudio := func(model string) map[string]any {
return map[string]any{
"model": model,
"messages": []any{map[string]any{
"role": "user",
"content": []any{map[string]any{
"type": "input_audio",
"input_audio": map[string]any{"data": "data:audio/wav;base64,UklGRg=="},
}},
}},
}
}
cases := []struct {
name, provider, model, base string
wantPath string
wantFields map[string]string
wantBody map[string]any
wantSegments int
}{
{
name: "openai whisper-1 serves segments", provider: "openai", model: "whisper-1", base: "/v1",
wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "whisper-1", "response_format": "verbose_json"},
wantSegments: 1,
},
{
name: "openai gpt-4o-transcribe accepts only json", provider: "openai", model: "gpt-4o-transcribe",
base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "gpt-4o-transcribe"},
},
{
name: "openai model not yet in the catalog", provider: "openai", model: "gpt-5-transcribe",
base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "gpt-5-transcribe"},
},
{
name: "siliconflow documents file and model only", provider: "siliconflow",
model: "FunAudioLLM/SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "FunAudioLLM/SenseVoiceSmall"},
},
{
name: "gpustack stays on json for its FunASR backend", provider: "gpustack",
model: "SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "SenseVoiceSmall"},
},
{
name: "generic stays on json", provider: "generic", model: "whisper-large-v3",
base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "whisper-large-v3"},
},
{
name: "zhipu glm-asr on the OpenAI shape", provider: "zhipu", model: "glm-asr-2512",
base: "/api/paas/v4", wantPath: "/api/paas/v4/audio/transcriptions",
wantFields: map[string]string{"model": "glm-asr-2512"},
},
{
name: "minimax on its own path", provider: "minimax", model: "asr-1.0",
base: "/v1", wantPath: "/v1/speech_to_text",
wantFields: map[string]string{"model": "asr-1.0"},
},
{
name: "openrouter", provider: "openrouter", model: "openai/whisper-large-v3",
base: "/api/v1", wantPath: "/api/v1/audio/transcriptions",
wantFields: map[string]string{"model": "openai/whisper-large-v3"},
},
{
name: "requesty whisper-1 serves segments", provider: "requesty", model: "openai/whisper-1",
base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "openai/whisper-1", "response_format": "verbose_json"},
wantSegments: 1,
},
{
name: "litellm proxy", provider: "litellm", model: "whisper",
base: "/v1", wantPath: "/v1/audio/transcriptions",
wantFields: map[string]string{"model": "whisper"},
},
{
name: "aliyun qwen3-asr-flash is served on chat", provider: "aliyun", model: "qwen3-asr-flash",
base: "/compatible-mode/v1", wantPath: "/compatible-mode/v1/chat/completions",
wantBody: chatAudio("qwen3-asr-flash"),
},
{
name: "mimo asr is served on chat", provider: "mimo", model: "mimo-v2.5-asr",
base: "/v1", wantPath: "/v1/chat/completions",
wantBody: chatAudio("mimo-v2.5-asr"),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
url, got, _ := upstream(t)
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,
ModelName: tc.model, APIKey: "k",
})
require.NoError(t, err)
out, err := a.Transcribe(context.Background(), []byte("RIFF"), "meeting.wav")
require.NoError(t, err)
assert.Equal(t, tc.wantPath, got.path)
assert.Equal(t, "Bearer k", got.auth)
if tc.wantBody != nil {
assert.Equal(t, tc.wantBody, got.body)
} else {
assert.Equal(t, tc.wantFields, got.fields)
assert.Equal(t, "meeting.wav", got.fileName)
}
assert.Equal(t, "hello", out.Text)
assert.Len(t, out.Segments, tc.wantSegments)
})
}
}
// A file over the documented ceiling is refused before it is uploaded: the
// vendor would reject it anyway, after the whole upload.
func TestOversizedAudioIsRefusedBeforeUpload(t *testing.T) {
url, _, calls := upstream(t)
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1",
ModelName: "whisper-1", APIKey: "k",
})
require.NoError(t, err)
_, err = a.Transcribe(context.Background(), make([]byte, 25<<20+1), "long.mp3")
require.Error(t, err)
assert.Contains(t, err.Error(), "at most 25 MB")
assert.Zero(t, calls.Load(), "nothing should have been sent")
_, err = a.Transcribe(context.Background(), make([]byte, 25<<20), "exactly.mp3")
require.NoError(t, err, "the ceiling itself is allowed")
}
// The chat-served recognisers cap the data URI as sent at 10 MB, prefix
// included, so three quarters of 10 MB of audio is already too much.
func TestChatServedCeilingCountsTheWholeDataURI(t *testing.T) {
url, _, calls := upstream(t)
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "aliyun", BaseURL: url + "/compatible-mode/v1",
ModelName: "qwen3-asr-flash", APIKey: "k",
})
require.NoError(t, err)
_, err = a.Transcribe(context.Background(), make([]byte, 10<<20*3/4), "long.wav")
require.Error(t, err)
assert.Zero(t, calls.Load())
prefix := len("data:audio/wav;base64,")
_, err = a.Transcribe(context.Background(), make([]byte, (10<<20-prefix)/4*3), "fits.wav")
require.NoError(t, err)
}
// Only qwen3-asr-flash takes the audio in the request. Alibaba's other
// recognition models want a public URL or an asynchronous task, so a row
// naming one is refused with the reason rather than sent a chat request.
func TestAlibabaRecognitionModelsOtherThanQwenASRFlashAreRefused(t *testing.T) {
_, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "aliyun",
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", ModelName: "paraformer-v2", APIKey: "k",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "public file URL")
}
// A vendor that does not declare speech recognition is not trusted with it.
// Azure's Endpoint hook still knows a transcription path, but nobody has
// verified it; a row that names Azure is refused, and a row that names no
// vendor gets what the pre-catalog client sent: the OpenAI shape at its URL.
func TestVendorsWithoutASRAreNotRoutedThroughTheirHooks(t *testing.T) {
_, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "azure_openai",
BaseURL: "https://example.openai.azure.com", ModelName: "whisper", APIKey: "k",
})
require.Error(t, err)
// Detection matches URL patterns as substrings, so a path carrying Azure's
// host detects as Azure while the request still reaches the test server.
url, got, _ := upstream(t)
base := url + "/openai.azure.com/v1"
require.Equal(t, "azure_openai", catalog.DetectByURL(base))
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, BaseURL: base, ModelName: "whisper", APIKey: "k",
})
require.NoError(t, err)
_, err = a.Transcribe(context.Background(), []byte("RIFF"), "a.wav")
require.NoError(t, err)
assert.Equal(t, "/openai.azure.com/v1/audio/transcriptions", got.path, "not Azure's /openai/v1 path")
assert.Equal(t, "Bearer k", got.auth, "not Azure's api-key header")
}
// A format the vendor does not list is refused before upload.
func TestUndocumentedFormatIsRefusedBeforeUpload(t *testing.T) {
url, _, calls := upstream(t)
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "zhipu", BaseURL: url + "/api/paas/v4",
ModelName: "glm-asr-2512", APIKey: "k",
})
require.NoError(t, err)
_, err = a.Transcribe(context.Background(), []byte("x"), "memo.m4a")
require.Error(t, err)
assert.Contains(t, err.Error(), "wav/mp3")
assert.Zero(t, calls.Load())
}
// The knowledge base's language hint reaches each vendor where its reference
// puts it, and nowhere for a vendor that documents none.
func TestLanguageHintGoesWhereEachVendorDocumentsIt(t *testing.T) {
for _, tc := range []struct {
name, provider, model, base string
check func(t *testing.T, got *form)
}{
{"openai form field", "openai", "whisper-1", "/v1", func(t *testing.T, got *form) {
assert.Equal(t, "zh", got.fields["language"])
assert.Empty(t, got.language)
}},
{"minimax header", "minimax", "asr-1.0", "/v1", func(t *testing.T, got *form) {
assert.Equal(t, "zh", got.language)
assert.NotContains(t, got.fields, "language")
}},
{"aliyun asr_options", "aliyun", "qwen3-asr-flash", "/compatible-mode/v1", func(t *testing.T, got *form) {
assert.Equal(t, map[string]any{"language": "zh"}, got.body["asr_options"])
}},
{
"siliconflow documents none", "siliconflow", "FunAudioLLM/SenseVoiceSmall", "/v1",
func(t *testing.T, got *form) {
assert.NotContains(t, got.fields, "language")
assert.Empty(t, got.language)
},
},
} {
t.Run(tc.name, func(t *testing.T) {
url, got, _ := upstream(t)
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,
ModelName: tc.model, APIKey: "k",
})
require.NoError(t, err)
_, err = a.Transcribe(WithLanguage(context.Background(), "zh"), []byte("RIFF"), "a.wav")
require.NoError(t, err)
tc.check(t, got)
})
}
// "auto" is every vendor's default and is not sent.
url, got, _ := upstream(t)
a, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1", ModelName: "whisper-1", APIKey: "k",
})
require.NoError(t, err)
_, err = a.Transcribe(WithLanguage(context.Background(), "auto"), []byte("RIFF"), "a.wav")
require.NoError(t, err)
assert.NotContains(t, got.fields, "language")
}
func TestEmptyAudioIsAnError(t *testing.T) {
url, _, calls := upstream(t)
a, err := NewASR(&Config{Source: types.ModelSourceRemote, Provider: "generic", BaseURL: url, ModelName: "m"})
require.NoError(t, err)
_, err = a.Transcribe(context.Background(), nil, "a.wav")
assert.Error(t, err)
assert.Zero(t, calls.Load())
}
func TestInternalBaseURLIsRefused(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "")
secutils.ResetSSRFWhitelistForTest()
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
_, err := NewASR(&Config{
Source: types.ModelSourceRemote, Provider: "generic",
BaseURL: "http://169.254.169.254/latest/meta-data/", ModelName: "asr-test",
})
assert.Error(t, err)
}

View File

@ -346,6 +346,8 @@ type VendorCompat struct {
Rerank RerankCompat `json:"rerank,omitempty"`
// Embeddings is the embedding protocol overlay, likewise one per vendor.
Embeddings EmbeddingsCompat `json:"embeddings,omitempty"`
// Transcriptions is the speech-to-text overlay, likewise one per vendor.
Transcriptions TranscriptionsCompat `json:"transcriptions,omitempty"`
}
// apply writes the set fields of an overlay struct onto a settings struct

View File

@ -34,6 +34,9 @@ func Register(v *Vendor) {
if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) {
v.RerankAPI = api.RerankCohere
}
if v.TranscriptionAPI == "" && v.SupportsType(types.ModelTypeASR) {
v.TranscriptionAPI = api.TranscriptionOpenAI
}
if v.EmbeddingAPI == "" && v.SupportsType(types.ModelTypeEmbedding) {
v.EmbeddingAPI = api.EmbeddingOpenAI
}

View File

@ -81,6 +81,11 @@ type Resolved struct {
// for the same reason.
EmbeddingAPI api.EmbeddingAPI
Embeddings EmbeddingsSettings
// TranscriptionAPI and Transcriptions are filled only for an ASR
// reference.
TranscriptionAPI api.TranscriptionAPI
Transcriptions TranscriptionsSettings
}
// Resolve merges the vendor, catalog entry, extra-config and per-row
@ -122,6 +127,9 @@ func Resolve(ref Ref) (*Resolved, error) {
if modelType == types.ModelTypeEmbedding {
return resolveEmbeddings(ref, vendor, spec, cataloged, baseURL)
}
if modelType == types.ModelTypeASR {
return resolveTranscriptions(ref, vendor, spec, cataloged, baseURL)
}
resolvedAPI := spec.API
if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" {
@ -537,3 +545,62 @@ func resolveEmbeddings(
}
return out, nil
}
// resolveTranscriptions merges the speech-to-text layers, in the same order
// as resolveEmbeddings: protocol default, vendor compat, the catalog entry,
// the row's own compat, then remote_model_name.
func resolveTranscriptions(
ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string,
) (*Resolved, error) {
protocol := vendor.TranscriptionAPI
if protocol == "" {
protocol = api.TranscriptionOpenAI
}
settings := DefaultTranscriptions()
apply(&settings, &vendor.Compat.Transcriptions)
for _, raw := range []json.RawMessage{spec.Compat, ref.Override.CompatJSON()} {
if len(raw) == 0 {
continue
}
overlay := &TranscriptionsCompat{}
if err := decodeCompat(raw, overlay); err != nil {
return nil, fmt.Errorf("transcriptions compat: %w", err)
}
apply(&settings, overlay)
}
if settings.API != "" {
protocol = settings.API
}
if !protocol.Known() {
return nil, fmt.Errorf("catalog: unknown transcription api %q for %s/%s", protocol, vendor.ID, spec.ID)
}
settings.API = protocol
switch settings.LanguageParam {
case "", LanguageForm, LanguageHeader, LanguageASROptions:
default:
return nil, fmt.Errorf("catalog: unknown language_param %q for %s/%s",
settings.LanguageParam, vendor.ID, spec.ID)
}
// Checked after every layer: an entry may lift a vendor-wide refusal for
// the one model that takes the audio in the request.
if settings.UnsupportedReason != "" {
return nil, fmt.Errorf(
"catalog: %s does not serve %q through a protocol this build implements: %s",
vendor.ID, spec.ID, settings.UnsupportedReason,
)
}
out := &Resolved{
Vendor: vendor,
Spec: spec,
Cataloged: cataloged,
BaseURL: baseURL,
RemoteModel: ref.Model,
TranscriptionAPI: protocol,
Transcriptions: settings,
}
if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" {
out.RemoteModel = override
}
return out, nil
}

View File

@ -0,0 +1,70 @@
package catalog
import "github.com/Tencent/WeKnora/internal/models/api"
// Where a transcription protocol puts the operator's language hint. Each
// vendor documents one place, or none.
const (
// LanguageForm is a multipart field named language (OpenAI's shape,
// ISO-639-1).
LanguageForm = "form"
// LanguageHeader is an HTTP request header named language (MiniMax,
// BCP-47).
LanguageHeader = "header"
// LanguageASROptions is asr_options.language in a chat-served request
// body (Alibaba, Xiaomi).
LanguageASROptions = "asr_options"
)
// TranscriptionsCompat is the overlay form (every field optional) of the
// speech-to-text settings. JSON keys are the names used in models.json.
type TranscriptionsCompat struct {
API *api.TranscriptionAPI `json:"api,omitempty"`
Path *string `json:"path,omitempty"`
// ResponseFormat is sent as response_format when non-empty. OpenAI's
// gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json, which is
// also the default, so most models leave it unset.
ResponseFormat *string `json:"response_format,omitempty"`
// LanguageParam names where the language hint goes: form, header or
// asr_options; empty sends none.
LanguageParam *string `json:"language_param,omitempty"`
// MaxFileBytes is the documented upload ceiling for the audio itself.
MaxFileBytes *int `json:"max_file_bytes,omitempty"`
// MaxEncodedBytes is the documented ceiling on the base64 data URI, for
// the protocols that send the audio that way.
MaxEncodedBytes *int `json:"max_encoded_bytes,omitempty"`
// Formats lists the file extensions the vendor documents, without the
// dot. Empty means the vendor documents no closed list.
Formats []string `json:"formats,omitempty"`
RequestTimeout *int `json:"request_timeout_seconds,omitempty"`
// UnsupportedReason marks a model this build cannot call.
UnsupportedReason *string `json:"unsupported_reason,omitempty"`
}
// TranscriptionsSettings is the resolved (fully defaulted) form.
type TranscriptionsSettings struct {
API api.TranscriptionAPI
Path string
ResponseFormat string
LanguageParam string
// MaxFileBytes and MaxEncodedBytes refuse an upload the vendor would
// reject anyway, before sending it; 0 leaves the check to the vendor.
MaxFileBytes int
MaxEncodedBytes int
Formats []string
// RequestTimeout caps one request, in seconds.
RequestTimeout int
// UnsupportedReason is set where a vendor's recognition models, or some
// of them, cannot be called with the audio in the request — they take a
// public URL or run as asynchronous tasks. Resolve refuses such a model
// rather than sending it a shape it does not accept.
UnsupportedReason string
}
// DefaultTranscriptions is the protocol baseline: a form with file and model
// and nothing else, which every speech-to-text endpoint in this catalog
// documents. The deadline is the one the pre-catalog client carried; audio
// transcription is slow.
func DefaultTranscriptions() TranscriptionsSettings {
return TranscriptionsSettings{Path: "/audio/transcriptions", RequestTimeout: 300}
}

View File

@ -251,6 +251,9 @@ type Vendor struct {
// OpenAI shape for any vendor that serves embeddings without naming
// another.
EmbeddingAPI api.EmbeddingAPI
// TranscriptionAPI is the speech-to-text protocol. Register defaults it
// to the OpenAI shape for any vendor that serves ASR.
TranscriptionAPI api.TranscriptionAPI
// DefaultBaseURLs by model type; GetDefaultURL falls back to chat.
DefaultBaseURLs map[types.ModelType]string
ModelTypes []types.ModelType

View File

@ -14,10 +14,9 @@ import (
// protocol, a bad compat key or an invalid thinking level is rejected where
// it is written instead of surfacing as a failed chat call much later.
//
// Chat, VLM, embedding and rerank rows are checked; all four are built from
// catalog.Resolve, so a row that does not resolve cannot be used. ASR rows
// return nil: the ASR client does not resolve through the catalog yet, so a
// spec stored on such a row is inert — neither validated nor applied.
// Every model type is checked: all of them are built from catalog.Resolve,
// so a row that does not resolve cannot be used and should fail the save
// rather than the first call.
func ValidateRow(modelName string, modelType types.ModelType, params *types.ModelParameters) error {
if params == nil || !resolvesThroughCatalog(modelType) {
return nil
@ -44,7 +43,8 @@ func ValidateRow(modelName string, modelType types.ModelType, params *types.Mode
func resolvesThroughCatalog(modelType types.ModelType) bool {
switch modelType {
case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank:
case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank,
types.ModelTypeASR:
return true
}
return false

View File

@ -56,9 +56,17 @@ func TestValidateRow(t *testing.T) {
wantErr: true,
},
{
name: "asr rows are not resolved through the catalog", modelName: "whisper-1",
name: "a catalogued asr row resolves", modelName: "whisper-1",
modelType: types.ModelTypeASR,
params: &types.ModelParameters{Provider: "openai", Spec: &types.ModelSpecOverride{API: "nope"}},
params: &types.ModelParameters{Provider: "openai"},
},
{
name: "unknown asr compat key is rejected", modelName: "whisper-1",
modelType: types.ModelTypeASR,
params: &types.ModelParameters{
Provider: "openai", Spec: &types.ModelSpecOverride{Compat: map[string]any{"dimensions_field": "x"}},
},
wantErr: true,
},
{
name: "a catalogued chat row resolves", modelName: "deepseek-v4-pro",

View File

@ -55,6 +55,8 @@
{"name": "qwen2.5-vl-embedding (fused vectors only)", "match": "qwen2.5-vl-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "dimension", "max_batch_size": 1}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},
{"name": "multimodal-embedding-v1", "match": "multimodal-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "", "max_batch_size": 20}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},
{"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
{"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}
{"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
{"id": "qwen3-asr-flash", "name": "Qwen3 ASR Flash", "type": "ASR", "compat": {"api": "openai-chat-audio", "max_encoded_bytes": 10485760, "language_param": "asr_options"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"},
{"name": "Other Alibaba recognition models (not callable with the audio in the request)", "match": "*", "type": "ASR", "compat": {"unsupported_reason": "Alibaba's other recognition models take a public file URL or run as asynchronous tasks; only qwen3-asr-flash accepts the audio in the request"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"}
]
}

View File

@ -136,6 +136,7 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
RerankAPI: api.RerankDashScope,
// Text and multimodal embeddings live under different roots of the
@ -160,6 +161,13 @@ func init() {
return root + "/compatible-mode/v1/embeddings", nil
},
Compat: catalog.VendorCompat{
// ASR: qwen3-asr-flash is the one recognition model callable with
// the audio in the request, on the compatible chat endpoint as a
// base64 data URI; its catalog entry declares that. Every other
// ASR name falls to a catch-all entry that refuses it: Paraformer,
// Fun-ASR and the *-filetrans models are asynchronous tasks that
// take a public file URL, which no protocol here can send
// (https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference).
// Text models: the OpenAI-compatible endpoint
// (https://help.aliyun.com/zh/model-studio/embedding-interfaces-compatible-with-openai),
// which takes model, input, dimensions and encoding_format.

View File

@ -124,14 +124,16 @@ func init() {
types.ModelTypeEmbedding: BaseURL,
types.ModelTypeVLLM: BaseURL,
},
// ASR is deliberately absent: Azure serves Whisper, but
// internal/models/asr builds every remote client with a plain
// OpenAI config (Bearer auth, {base}/audio/transcriptions) and has
// no provider or extra-config field to route through this vendor's
// api-key header and deployments path. Advertising the type would
// let the picker create a row that fails at first use. The Endpoint
// hook below already knows the ASR path, so re-adding the type is a
// one-line change once asr.Config carries the provider.
// ASR is deliberately absent. The ASR client now resolves through
// this vendor (api-key header, Endpoint hook below), but the only
// v1 reference for audio is the preview one —
// POST {endpoint}/openai/v1/audio/transcriptions?api-version=preview
// (https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest)
// — while the hook sends a row without api_version to the v1 path
// with no version at all. Until that is verified against a live
// resource, advertising the type would let the picker create a row
// that may fail at first use. A row with an explicit api_version
// would take the documented deployments path.
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,

View File

@ -108,6 +108,17 @@ func init() {
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
// Transcriptions keeps the baseline, json by default. The
// pre-catalog client always asked for verbose_json, which not every
// model or server can produce: OpenAI's gpt-4o transcribers accept
// only json, and vox-box's FunASR backend answers verbose_json with
// a bare JSON string. A row that wants segments sets
// {"response_format": "verbose_json"} in its compat. The language
// hint goes as OpenAI's language form field, which a server that
// does not implement it ignores.
Transcriptions: catalog.TranscriptionsCompat{
LanguageParam: catalog.Ptr(catalog.LanguageForm),
},
Embeddings: catalog.EmbeddingsCompat{
// Whatever the operator runs. Everything the pre-catalog client
// sent stays: vLLM, SGLang, TEI and Ollama's OpenAI route all

View File

@ -118,6 +118,17 @@ func init() {
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
// Transcriptions keeps the baseline, json by default. GPUStack
// serves audio through vox-box (https://github.com/gpustack/vox-box),
// whose route wraps a json answer as {"text": ...} but returns
// verbose_json as whatever the backend produced — and its FunASR
// backend (SenseVoice, Paraformer) produces a bare string for every
// format. The pre-catalog client's verbose_json therefore arrived
// as a JSON string that could not be decoded. The route reads a
// language form field.
Transcriptions: catalog.TranscriptionsCompat{
LanguageParam: catalog.Ptr(catalog.LanguageForm),
},
Embeddings: catalog.EmbeddingsCompat{
// vLLM's embedding server: dimensions for Matryoshka models, the
// row's truncation budget, and encoding_format.

View File

@ -77,6 +77,7 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
ExtraFields: []catalog.ExtraField{{
Key: catalog.ExtraScoreScale,
@ -104,6 +105,9 @@ func init() {
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
// Transcriptions keeps the baseline: the proxy serves the OpenAI
// /audio/transcriptions route for whichever upstream it is
// configured with (https://docs.litellm.ai/docs/audio_transcription).
Embeddings: catalog.EmbeddingsCompat{
// https://docs.litellm.ai/docs/embedding/supported_embedding:
// model, input, user, dimensions, encoding_format; anything else

View File

@ -5,6 +5,7 @@
{"id": "mimo-v2.5-pro", "name": "MiMo V2.5 Pro", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.435, "output": 0.87, "cache_read": 0.0036}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},
{"id": "mimo-v2.5", "name": "MiMo V2.5", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.14, "output": 0.28, "cache_read": 0.0028}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},
{"id": "mimo-v2.5-pro-ultraspeed", "name": "MiMo V2.5 Pro Ultraspeed", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 1.305, "output": 2.61, "cache_read": 0.0108}, "source": "https://mimo.mi.com/static/docs/news/latest/beta-extended.md"},
{"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"}
{"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"},
{"id": "mimo-v2.5-asr", "name": "MiMo V2.5 ASR", "type": "ASR", "source": "https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition"}
]
}

View File

@ -71,10 +71,22 @@ func init() {
DefaultBaseURLs: map[types.ModelType]string{
types.ModelTypeKnowledgeQA: BaseURL,
},
TranscriptionAPI: api.TranscriptionChatAudio,
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
// ASR: mimo-v2.5-asr on the chat endpoint, WAV or MP3 as a base64
// data URI whose "encoded string size must not exceed 10 MB"
// (https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition).
Transcriptions: catalog.TranscriptionsCompat{
// The ceiling counts the whole data URI as sent.
MaxEncodedBytes: catalog.Ptr(10 << 20),
Formats: []string{"wav", "mp3"},
// asr_options.language: "auto|zh|en".
LanguageParam: catalog.Ptr(catalog.LanguageASROptions),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
PromptCacheAccounting: catalog.Ptr(true),

View File

@ -9,6 +9,7 @@
{"id": "MiniMax-M2.5-highspeed", "name": "MiniMax M2.5 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.6, "output": 2.4, "cache_read": 0.06, "cache_write": 0.375}, "thinking_levels": {"off": null}},
{"id": "MiniMax-M2.1", "name": "MiniMax M2.1", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.03, "cache_write": 0.375}, "thinking_levels": {"off": null}},
{"id": "MiniMax-M2.1-highspeed", "name": "MiniMax M2.1 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "thinking_levels": {"off": null}, "source": "https://platform.minimax.io/docs/guides/text-generation"},
{"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}}
{"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}},
{"id": "asr-1.0", "name": "MiniMax ASR 1.0", "type": "ASR", "source": "https://platform.minimax.io/docs/api-reference/speech-to-text"}
]
}

View File

@ -95,8 +95,23 @@ func init() {
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
Transcriptions: catalog.TranscriptionsCompat{
// https://platform.minimax.io/docs/api-reference/speech-to-text
// (and the same page on platform.minimax.cn): multipart model
// (asr-1.0) + file on /v1/speech_to_text rather than the OpenAI
// path, answering {text, duration, trace_id} for the default
// json. At most 50 MB and 500 seconds; "超出会返回 400 而不会被截断".
Path: catalog.Ptr("/speech_to_text"),
MaxFileBytes: catalog.Ptr(50 << 20),
// "wav / aiff / flac / alac(m4a) / mp3 / aac / opus / ogg";
// "不支持无容器的裸 PCM 数据". The language hint is a request
// header, a BCP-47 tag, not a form field.
Formats: []string{"wav", "aiff", "flac", "m4a", "mp3", "aac", "opus", "ogg"},
LanguageParam: catalog.Ptr(catalog.LanguageHeader),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
ThinkingEnabledValue: catalog.Ptr("adaptive"),

View File

@ -29,6 +29,10 @@
{"id": "text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8191, "cost": {"input": 0.02, "output": 0}},
{"id": "text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8191, "cost": {"input": 0.13, "output": 0}},
{"id": "whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
{"id": "gpt-transcribe", "name": "GPT Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
{"id": "gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
{"id": "gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
{"id": "text-embedding-ada-002", "name": "text-embedding-ada-002", "type": "Embedding", "dimension": 1536, "context_window": 8192, "cost": {"input": 0.1, "output": 0}, "compat": {"dimensions_field": ""}}
]
}

View File

@ -97,6 +97,19 @@ func init() {
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
Transcriptions: catalog.TranscriptionsCompat{
// The transcription reference (URL in openaitranscriptions):
// response_format differs per model — "For gpt-4o-transcribe
// and gpt-4o-mini-transcribe, the only supported format is
// json", the default — so it is declared per entry, not here.
// "Files can be up to 25 MB"
// (https://developers.openai.com/api/docs/guides/speech-to-text).
MaxFileBytes: catalog.Ptr(25 << 20),
// "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm", and a
// language field in ISO-639-1.
Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},
LanguageParam: catalog.Ptr(catalog.LanguageForm),
},
Embeddings: catalog.EmbeddingsCompat{
// https://developers.openai.com/api/reference/resources/embeddings/methods/create:
// model, input, dimensions (text-embedding-3 and later; the ada-002

View File

@ -19,6 +19,7 @@
{"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}},
{"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
{"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}
{"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"},
{"id": "openai/whisper-large-v3", "name": "Whisper Large v3", "type": "ASR", "source": "https://openrouter.ai/docs/api/api-reference/stt/create-transcription"}
]
}

View File

@ -75,8 +75,18 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
Transcriptions: catalog.TranscriptionsCompat{
// https://openrouter.ai/docs/api/api-reference/stt/create-transcription:
// OpenAI-style multipart file + model, json by default. "Max 25
// MB; send larger files as base64 JSON via input_audio."
MaxFileBytes: catalog.Ptr(25 << 20),
// language is an ISO-639-1 form field. The format "is derived
// from the filename extension", with no closed list.
LanguageParam: catalog.Ptr(catalog.LanguageForm),
},
Embeddings: catalog.EmbeddingsCompat{
// https://openrouter.ai/docs/api/api-reference/embeddings/create-embeddings:
// model, input, dimensions, encoding_format, input_type, provider,

View File

@ -12,6 +12,9 @@
{"id": "google/gemini-3.6-flash", "name": "Gemini 3.6 Flash", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 65536, "thinking_levels": {"off": null, "minimal": "min", "xhigh": "high", "max": "high"}},
{"id": "deepseek/deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro", "reasoning": true, "input": ["text"], "context_window": 1000000, "max_output_tokens": 131072},
{"id": "zai/glm-5.2", "name": "GLM-5.2", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072},
{"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144}
{"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144},
{"id": "openai/gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},
{"id": "openai/gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},
{"id": "openai/whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"}
]
}

View File

@ -77,8 +77,19 @@ func init() {
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
Transcriptions: catalog.TranscriptionsCompat{
// https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create:
// OpenAI-compatible multipart, json by default, verbose_json
// available. "The maximum upload size per request is 32 MB."
MaxFileBytes: catalog.Ptr(32 << 20),
// "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, webm"; language
// is an ISO 639-1 form field.
Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},
LanguageParam: catalog.Ptr(catalog.LanguageForm),
},
Embeddings: catalog.EmbeddingsCompat{
// https://docs.requesty.ai/api-reference/endpoint/embeddings-create:
// model, input, encoding_format, dimensions.

View File

@ -24,6 +24,7 @@
{"id": "Qwen/Qwen3-Embedding-0.6B", "name": "Qwen3 Embedding 0.6B", "type": "Embedding", "dimension": 1024, "source": "https://api-docs.siliconflow.cn/docs/api/embeddings-post"},
{"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker v2 M3", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},
{"id": "Qwen/Qwen3-Reranker-8B", "name": "Qwen3 Reranker 8B", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},
{"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://docs.siliconflow.cn/cn/api-reference/audio/create-audio-transcriptions"}
{"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"},
{"id": "TeleAI/TeleSpeechASR", "name": "TeleSpeechASR", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"}
]
}

View File

@ -77,6 +77,12 @@ func init() {
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
Transcriptions: catalog.TranscriptionsCompat{
// https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post:
// file and model only — no response_format — and a file of
// "时长不超过 1 小时,文件大小不超过 50MB".
MaxFileBytes: catalog.Ptr(50 << 20),
},
Embeddings: catalog.EmbeddingsCompat{
// https://api-docs.siliconflow.cn/docs/api/embeddings-post: model,
// input, encoding_format, and dimensions "仅 Qwen/Qwen3 系列支持"

View File

@ -110,6 +110,9 @@ func TestEveryCatalogEntryResolves(t *testing.T) {
if modelType == types.ModelTypeKnowledgeQA && !r.API.Known() {
t.Errorf("%s/%s: resolved to unknown API %q", id, name, r.API)
}
if modelType == types.ModelTypeASR && !r.TranscriptionAPI.Known() {
t.Errorf("%s/%s: resolved to unknown transcription API %q", id, name, r.TranscriptionAPI)
}
if modelType == types.ModelTypeEmbedding && !r.EmbeddingAPI.Known() {
t.Errorf("%s/%s: resolved to unknown embedding API %q", id, name, r.EmbeddingAPI)
}

View File

@ -15,6 +15,7 @@
{"id": "glm-4.5-air", "name": "GLM-4.5 Air", "reasoning": true, "input": ["text"], "context_window": 131072, "max_output_tokens": 98304, "cost": {"input": 0.2, "output": 1.1, "cache_read": 0.03}, "compat": {"supports_reasoning_effort": false}},
{"id": "embedding-3", "name": "Embedding-3", "type": "Embedding", "dimension": 2048, "compat": {"max_batch_size": 64}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},
{"id": "embedding-2", "name": "Embedding-2", "type": "Embedding", "dimension": 1024, "compat": {"dimensions_field": ""}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},
{"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"}
{"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"},
{"id": "glm-asr-2512", "name": "GLM-ASR-2512 (at most 30 s per file)", "type": "ASR", "source": "https://docs.bigmodel.cn/api-reference/模型-api/语音转文本"}
]
}

View File

@ -107,8 +107,19 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
Transcriptions: catalog.TranscriptionsCompat{
// https://docs.bigmodel.cn/api-reference/模型-api/语音转文本:
// POST {base}/audio/transcriptions, multipart file + model
// (glm-asr-2512), answering {text}. The file is ".wav / .mp3",
// at most 25 MB and 30 seconds — the duration cannot be checked
// here without decoding, so a longer file is the vendor's error.
MaxFileBytes: catalog.Ptr(25 << 20),
// No language field; prompt and hotwords are its only hints.
Formats: []string{"wav", "mp3"},
},
Embeddings: catalog.EmbeddingsCompat{
// https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入: model,
// input, dimensions — no encoding_format. embedding-2 is fixed at

View File

@ -44,44 +44,3 @@ func ApplyCustomHeaders(req *http.Request, headers map[string]string) {
req.Header.Set(name, v)
}
}
// CustomHeadersRoundTripper 是一个 http.RoundTripper 包装器,
// 会在每个 HTTP 请求发出前注入用户自定义的 header。
// 用于无法直接拿到底层 *http.Request 的场景(如 go-openai SDK)。
type CustomHeadersRoundTripper struct {
Headers map[string]string
Base http.RoundTripper
}
// RoundTrip 实现 http.RoundTripper 接口。
func (t *CustomHeadersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
base := t.Base
if base == nil {
base = http.DefaultTransport
}
if len(t.Headers) > 0 {
// 复制请求以避免修改调用方传入的对象。
cloned := req.Clone(req.Context())
ApplyCustomHeaders(cloned, t.Headers)
return base.RoundTrip(cloned)
}
return base.RoundTrip(req)
}
// WrapHTTPClientWithHeaders 返回一个新的 *http.Client,在原有 client 基础上注入自定义 header。
// 如果 headers 为空则直接返回原 client,避免不必要的开销。
func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) *http.Client {
if len(headers) == 0 {
return client
}
if client == nil {
client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig())
}
base := client.Transport
if base == nil {
base = http.DefaultTransport
}
wrapped := *client
wrapped.Transport = &CustomHeadersRoundTripper{Headers: headers, Base: base}
return &wrapped
}

View File

@ -1,10 +1,7 @@
package utils
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
@ -48,47 +45,3 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) {
t.Fatalf("unexpected headers added: %+v", req.Header)
}
}
func TestWrapHTTPClientWithHeaders(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")
ResetSSRFWhitelistForTest()
t.Cleanup(ResetSSRFWhitelistForTest)
gotTrace := ""
gotAuth := ""
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotTrace = r.Header.Get("X-Trace-Id")
gotAuth = r.Header.Get("Authorization")
io.Copy(io.Discard, r.Body)
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
client := WrapHTTPClientWithHeaders(nil, map[string]string{
"X-Trace-Id": "rt-1",
"Authorization": "Bearer should-not-override",
})
req, _ := http.NewRequest("POST", srv.URL, strings.NewReader("{}"))
req.Header.Set("Authorization", "Bearer kept")
resp, err := client.Do(req)
if err != nil {
t.Fatalf("request failed: %v", err)
}
resp.Body.Close()
if gotTrace != "rt-1" {
t.Fatalf("expected custom header injected, got %q", gotTrace)
}
if gotAuth != "Bearer kept" {
t.Fatalf("reserved header must not be overridden, got %q", gotAuth)
}
}
func TestWrapHTTPClientWithHeaders_EmptyReturnsOriginal(t *testing.T) {
orig := &http.Client{}
wrapped := WrapHTTPClientWithHeaders(orig, nil)
if wrapped != orig {
t.Fatalf("expected original client returned when headers empty")
}
}

View File

@ -58,7 +58,7 @@ WeKnora 采用"主服务 + 前端 + 文档解析微服务"的三进程核心架
| 配置 | `github.com/spf13/viper` + `config/config.yaml` + 环境变量 | — |
| 可观测 | OpenTelemetry + Langfuse(`internal/tracing/langfuse`) | LLM 调用级 trace |
| gRPC | `google.golang.org/grpc` v1.81.0 | 调用 docreader |
| LLM 接入 | `sashabaranov/go-openai`、Ollama、腾讯云 LKE 等 | 18+ 模型提供商(OpenAI 兼容 / Ollama / 云厂商 SDK) |
| LLM 接入 | 自研协议层 `internal/models/api`(OpenAI / Anthropic / Gemini / DashScope 等线格式各一个包)、Ollama、腾讯云 LKE SDK 等 | 27 家厂商由 `internal/models/vendors` 声明,见[模型管理](../03-features/06-models.md) |
| 向量/检索 | pgvector、ES v7/v8、OpenSearch、Qdrant、Milvus、Weaviate、Doris、腾讯 VectorDB、sqlite-vec | 由 `RETRIEVE_DRIVER` 与 `vector_stores` 表动态装配 |
| 知识图谱 | `neo4j-go-driver/v6` | 可选 |
| 数据分析 | DuckDB(`duckdb-go/v2`)、`pg_query_go` SQL 校验 | Agent 数据分析工具 |

View File

@ -56,7 +56,7 @@ const (
| `Embedding` | `embedding` | `internal/models/embedding` | `Embed` / `BatchEmbed`(含 `GetDimensions`) | 文本向量化,供向量检索索引与查询 |
| `Rerank` | `rerank` | `internal/models/rerank` | `Rerank(query, documents)` 返回 `RankResult` | 检索结果精排 |
| `VLLM` | `vllm` | `internal/models/vlm` | `Predict(imgBytes, prompt)` | 视觉语言模型(VLM),文档图片理解 / 多模态解析 |
| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本与分段时间戳 | 音频转写(自动语音识别) |
| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本;模型提供时附带分段时间戳(如 OpenAI `whisper-1`) | 音频转写(自动语音识别) |
前后端类型映射见 `internal/handler/model.go` 的 `modelTypeToFrontend()`(`KnowledgeQA -> chat` 等)。
@ -176,7 +176,7 @@ builtin_models:
| 层 | 位置 | 职责 |
|----|------|------|
| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`。各自持有请求/响应结构与解析,不含任何厂商名 |
| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`;语音:`openaitranscriptions`。各自持有请求/响应结构与解析,不含任何厂商名 |
| 厂商层 | `internal/models/vendors/<id>/` | 一个厂商一个目录:`vendor.go`(注册 `catalog.Vendor`)、`models.json`(模型目录)、`icon.svg`(品牌图标,`go:embed` 进二进制) |
| 目录层 | `internal/models/catalog` | 合并厂商定义、模型条目、部署叠加与单行覆盖,`Resolve` 得出「这个模型到底怎么发请求」 |
@ -235,7 +235,7 @@ make model-catalog-check
前端不需要任何改动:厂商下拉、图标、额外字段、内置模型列表都由 `GET /api/v1/models/providers` 动态渲染。
写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(chat、VLM、embedding、rerank 四类;ASR 还没进目录),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。
写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(全部模型类型),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。
#### 厂商更新了模型怎么办
@ -280,7 +280,21 @@ Embedding 行也有几处按厂商文档纠正的行为变化(逐厂商的出
7. **OpenAI 兼容回复里没有 `index` 时按顺序取**(和旧实现一致);有 `index` 就按它放回,重复或缺位报错。
8. **Jina 的 `task`、Gemini 的 `taskType`、OpenRouter 的 `input_type`、火山的 `instructions`、百炼原生接口的 `text_type` / `instruct` 都不发**。它们会改变文档侧向量,开启后同一个知识库里新旧向量不在同一空间;需要按行显式开启的设计另见 Tencent/WeKnora#1401。
另外 Azure OpenAI 不再声明支持 ASR(ASR 客户端只会构造标准 OpenAI 客户端,根本无法带上 Azure 的 `api-key` 头和部署路径,这类行此前就调不通)。
ASR 行的变化(逐厂商的出站表单由 `internal/models/asr/wire_test.go` 钉住):
1. **不再一律发 `response_format=verbose_json`**。OpenAI 文档写明 gpt-4o-transcribe / gpt-4o-mini-transcribe「only supported format is json」,旧实现发给它们必然 400;GPUStack 的音频后端 vox-box 在 FunASR 模型(SenseVoice、Paraformer)上对 `verbose_json` 返回裸字符串,旧实现解不开。现在只有文档写明支持的模型声明它(OpenAI `whisper-1`),其余走默认 json,Langfuse 里的音频时长因此只在返回分段的模型上有。需要分段的自建行可以在 `spec.compat` 里写 `{"response_format": "verbose_json"}`。
2. **按厂商文档的上限和格式在上传前拒绝**:文件大小(OpenAI / 智谱 / OpenRouter 25 MB、Requesty 32 MB、SiliconFlow / MiniMax 50 MB;阿里与小米按整段 `data:` URI 计 10 MB),以及文档给出封闭格式清单的厂商(智谱、小米只收 wav/mp3,OpenAI、Requesty、MiniMax 各有列表)。
3. **回复里没有 `text` 字段即报错**,不再当成「未检测到语音」入库;静音音频返回的是空字符串 `text`,照常处理。
4. ASR 行现在带上 provider 走目录解析,go-openai 依赖随之移除。provider 没有声明 ASR 的行:显式填了该厂商的直接报错;没填、靠 URL 识别到的,按 OpenAI 形状打到它的 URL(与旧实现一致),不走该厂商的 Endpoint 钩子。
5. **支持 ASR 的厂商按文档补齐**。OpenAI 形状(multipart `file` + `model`):openai、siliconflow、gpustack、generic、智谱(`glm-asr-2512`,单文件 ≤30 秒)、MiniMax(`asr-1.0`,路径是 `/v1/speech_to_text`)、OpenRouter、Requesty、LiteLLM。经 chat completions 的 `input_audio`(base64 data URI,编码后 ≤10 MB):阿里云 `qwen3-asr-flash`、小米 `mimo-v2.5-asr`。
6. **知识库的「音频语言提示」终于发出去了**(此前从未接线)。位置按厂商文档:OpenAI / Requesty / OpenRouter / GPUStack / generic 是 `language` 表单字段,MiniMax 是 `language` 请求头,阿里与小米是 `asr_options.language`;智谱、SiliconFlow、LiteLLM 文档没有这个参数,不发。填 `auto` 等同留空。
7. **音频时长从回包里读**:MiniMax 的 `duration`、OpenAI 系与阿里的 `usage.seconds`,Langfuse 不再只靠 `verbose_json` 的分段。
阿里只有 `qwen3-asr-flash` 能直接带音频调用;其余 ASR 模型名由一条兜底条目拒绝并说明原因。
查过但没有接的:火山豆包语音(独立域名与密钥,当前文档的请求体只收音频 URL)、千帆(`vop_asr` 挂在应用实例下要 `app_id`;短语音接口 ≤60 秒且只收 pcm/wav/amr/m4a)、七牛(只收音频 URL)、Novita(GLM-ASR 自有接口,≤30 秒)、腾讯云 ASR(与混元不是同一套鉴权)、NVIDIA 托管 Riva(gRPC)、Gemini(没有专用转写接口,只能让对话模型听音频)、Azure(见下)。阿里 Paraformer / Fun-ASR / `*-filetrans` 是异步任务,同样要公网 URL。
Azure OpenAI 仍不声明 ASR:音频转写只出现在 v1 **preview** 参考里(`/openai/v1/audio/transcriptions?api-version=preview`),v1 GA 参考没有;未填 `api_version` 的行会被钩子发到不带版本的 v1 路径,能否调通没有核实。
### 模型调用链