refactor(asr): move the wire format into protocol packages, and cover every vendor whose docs offer ASR (#3503)
parent
7d688cb8d3
commit
4a9aa06274
2
Makefile
2
Makefile
|
|
@ -117,7 +117,7 @@ test:
|
|||
.PHONY: model-catalog-check
|
||||
model-catalog-check:
|
||||
go test ./internal/models/parity/ ./internal/models/vendors/ ./internal/models/catalog/ ./internal/models/api/... \
|
||||
./internal/models/rerank/ ./internal/models/embedding/
|
||||
./internal/models/rerank/ ./internal/models/embedding/ ./internal/models/asr/
|
||||
|
||||
# Vendor catalog: report where our model metadata differs from models.dev.
|
||||
# Development aid only — nothing is fetched at runtime and nothing is written
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -51,7 +51,6 @@ require (
|
|||
github.com/redis/go-redis/v9 v9.14.1
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
github.com/sashabaranov/go-openai v1.41.2
|
||||
github.com/sirupsen/logrus v1.9.4
|
||||
github.com/slack-go/slack v0.23.1
|
||||
github.com/spf13/viper v1.21.0
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -2516,8 +2516,6 @@ github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=
|
|||
github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM=
|
||||
github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/asr"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/searchutil"
|
||||
|
|
@ -3699,7 +3700,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
|||
return nil
|
||||
}
|
||||
|
||||
transcriptionResult, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
|
||||
// The knowledge base's language hint; the vendor declares where it
|
||||
// goes, or that it takes none.
|
||||
asrCtx := asr.WithLanguage(ctx, eff.ASRConfig.Language)
|
||||
transcriptionResult, err := asrModel.Transcribe(asrCtx, convertResult.AudioData, knowledge.FileName)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
|
||||
if isLastRetry {
|
||||
|
|
|
|||
|
|
@ -759,8 +759,7 @@ func (h *ModelHandler) DeleteModel(c *gin.Context) {
|
|||
// validateCatalogParameters rejects protocol / compat overrides the catalog
|
||||
// cannot interpret (unknown extra_config.api, unknown compat keys, bad
|
||||
// reasoning levels) so a typo fails at save time instead of at the first
|
||||
// call. Chat, VLM, embedding and rerank rows are checked; see
|
||||
// catalog.ValidateRow.
|
||||
// call. Every model type is checked; see catalog.ValidateRow.
|
||||
func validateCatalogParameters(name string, modelType types.ModelType, params *types.ModelParameters) error {
|
||||
return catalog.ValidateRow(name, modelType, params)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,24 +100,33 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b
|
|||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
if err := secutils.ValidateURLForSSRF(url); err != nil {
|
||||
return nil, nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
|
||||
req, err := e.newPost(ctx, url, data, "application/json")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create request: %w", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if stream {
|
||||
req.Header.Set("Accept", "text/event-stream")
|
||||
}
|
||||
return req, data, nil
|
||||
}
|
||||
|
||||
// newPost prepares an authenticated POST of an already-encoded body.
|
||||
func (e Endpoint) newPost(ctx context.Context, url string, data []byte, contentType string) (*http.Request, error) {
|
||||
if err := secutils.ValidateURLForSSRF(url); err != nil {
|
||||
return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
if e.Auth != nil {
|
||||
e.Auth(req, data)
|
||||
}
|
||||
// User headers are applied last but the helper skips reserved names so
|
||||
// they cannot clobber auth or content negotiation.
|
||||
secutils.ApplyCustomHeaders(req, e.Headers)
|
||||
return req, data, nil
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// Do sends the request and returns the response. Non-2xx responses are
|
||||
|
|
@ -125,7 +134,7 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b
|
|||
func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
|
||||
resp, err := e.httpClient().Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("send request: %w", err)
|
||||
return nil, &TransportError{Op: "send request", Err: err}
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
|
|
@ -151,6 +160,11 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
|
|||
// prompt is what an operator debugs, while a rerank body is a query plus
|
||||
// every candidate chunk. The rerank layer logs a truncated line at Debug
|
||||
// instead, so this would have duplicated it at Info.
|
||||
return e.roundTrip(req, out)
|
||||
}
|
||||
|
||||
// roundTrip sends a prepared request and decodes a 2xx reply into out.
|
||||
func (e Endpoint) roundTrip(req *http.Request, out any) error {
|
||||
resp, err := e.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -158,7 +172,9 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
|
|||
defer func() { _ = resp.Body.Close() }()
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read response: %w", err)
|
||||
// The connection broke before the reply arrived whole: nothing was
|
||||
// received, so this is the network failing, not the vendor answering.
|
||||
return &TransportError{Op: "read response", Err: err}
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
|
|
@ -169,6 +185,18 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
|
|||
return nil
|
||||
}
|
||||
|
||||
// TransportError is a request that never got a whole answer: DNS, connect,
|
||||
// TLS, a reset or a timeout, while sending or while reading the reply. It is
|
||||
// the only failure worth sending again.
|
||||
type TransportError struct {
|
||||
// Op is the phase that failed: "send request" or "read response".
|
||||
Op string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *TransportError) Error() string { return e.Op + ": " + e.Err.Error() }
|
||||
func (e *TransportError) Unwrap() error { return e.Err }
|
||||
|
||||
// HTTPError is a non-2xx vendor reply.
|
||||
type HTTPError struct {
|
||||
StatusCode int
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FormField is one text part of a multipart form. Order is kept.
|
||||
type FormField struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
// FormFile is the file part of a multipart form.
|
||||
type FormFile struct {
|
||||
Field string
|
||||
FileName string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// EncodeMultipart writes fields and one file into a multipart/form-data body
|
||||
// and returns it with its Content-Type. It is encoded once: those bytes are
|
||||
// what a signing vendor hashes and what every retry sends again.
|
||||
//
|
||||
// The file part carries a filename and no Content-Type of its own. Servers
|
||||
// identify the format from the extension then — OpenAI asks for "an
|
||||
// extension-bearing filename", and vox-box (GPUStack's audio backend) falls
|
||||
// back to guessing from it — whereas a guessed type that a server does not
|
||||
// list is rejected outright.
|
||||
func EncodeMultipart(fields []FormField, file FormFile) ([]byte, string, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
header := make(textproto.MIMEHeader)
|
||||
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`,
|
||||
file.Field, filepath.Base(file.FileName)))
|
||||
part, err := w.CreatePart(header)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("encode form file: %w", err)
|
||||
}
|
||||
if _, err := part.Write(file.Data); err != nil {
|
||||
return nil, "", fmt.Errorf("encode form file: %w", err)
|
||||
}
|
||||
for _, f := range fields {
|
||||
if err := w.WriteField(f.Name, f.Value); err != nil {
|
||||
return nil, "", fmt.Errorf("encode form field %s: %w", f.Name, err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, "", fmt.Errorf("encode form: %w", err)
|
||||
}
|
||||
return buf.Bytes(), w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// PostMultipartWithRetry sends an encoded multipart body, retrying transport
|
||||
// failures only, and decodes the JSON reply into out. It has the guarantees
|
||||
// of PostJSONWithRetry.
|
||||
func (e Endpoint) PostMultipartWithRetry(
|
||||
ctx context.Context, url string, body []byte, contentType string,
|
||||
out any, policy RetryPolicy, label string,
|
||||
) error {
|
||||
if !strings.HasPrefix(contentType, "multipart/") {
|
||||
return fmt.Errorf("%s: not a multipart content type: %q", label, contentType)
|
||||
}
|
||||
return withRetry(ctx, policy, label, func() error {
|
||||
req, err := e.newPost(ctx, url, body, contentType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return e.roundTrip(req, out)
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
// Package openaichataudio implements speech recognition served through the
|
||||
// chat completions endpoint: one user message whose only content part is
|
||||
// {"type": "input_audio", "input_audio": {"data": "data:<mime>;base64,..."}},
|
||||
// answered by an assistant message whose content is the transcript.
|
||||
//
|
||||
// Only dedicated recognition models are addressed this way — Alibaba's
|
||||
// qwen3-asr-flash and Xiaomi's mimo-v2.5-asr — so no instruction is sent: the
|
||||
// model transcribes whatever it is given. A general audio-capable chat model
|
||||
// would need a prompt and would answer about the audio rather than transcribe
|
||||
// it, which is why this is its own protocol and not a chat request.
|
||||
//
|
||||
// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference
|
||||
// https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition
|
||||
package openaichataudio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
)
|
||||
|
||||
// Config is everything the client needs, already resolved by the catalog.
|
||||
type Config struct {
|
||||
Endpoint api.Endpoint
|
||||
Settings catalog.TranscriptionsSettings
|
||||
Retry api.RetryPolicy
|
||||
}
|
||||
|
||||
// Client talks chat-served speech recognition to one endpoint.
|
||||
type Client struct {
|
||||
cfg Config
|
||||
}
|
||||
|
||||
// New builds a client.
|
||||
func New(cfg Config) *Client { return &Client{cfg: cfg} }
|
||||
|
||||
const defaultPath = "/chat/completions"
|
||||
|
||||
func (c *Client) url() string {
|
||||
if c.cfg.Endpoint.URL != "" {
|
||||
return c.cfg.Endpoint.Resolve("")
|
||||
}
|
||||
path := defaultPath
|
||||
if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
|
||||
path = ""
|
||||
}
|
||||
return c.cfg.Endpoint.Resolve(path)
|
||||
}
|
||||
|
||||
// audioTypes maps the file extension onto the media type of the data URI.
|
||||
// The two references show audio/wav and audio/mpeg; the rest are the
|
||||
// registered types for the containers Alibaba lists. A fixed table rather
|
||||
// than mime.TypeByExtension, which reads the host's tables and would make the
|
||||
// request depend on the machine it runs on.
|
||||
var audioTypes = map[string]string{
|
||||
".wav": "audio/wav",
|
||||
".mp3": "audio/mpeg",
|
||||
".m4a": "audio/mp4",
|
||||
".mp4": "audio/mp4",
|
||||
".aac": "audio/aac",
|
||||
".flac": "audio/flac",
|
||||
".ogg": "audio/ogg",
|
||||
".opus": "audio/opus",
|
||||
".webm": "audio/webm",
|
||||
".amr": "audio/amr",
|
||||
}
|
||||
|
||||
// DataURI is the input_audio payload for one file. Both references show an
|
||||
// audio media type in the URI, so a file whose extension names no audio
|
||||
// container is refused rather than labelled application/octet-stream.
|
||||
func DataURI(audio []byte, fileName string) (string, error) {
|
||||
mediaType, ok := audioTypes[strings.ToLower(filepath.Ext(fileName))]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("cannot tell the audio format of %q from its extension", fileName)
|
||||
}
|
||||
return "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(audio), nil
|
||||
}
|
||||
|
||||
// BuildRequestBody is the golden-test entry point.
|
||||
func (c *Client) BuildRequestBody(req api.TranscriptionRequest) (map[string]any, error) {
|
||||
uri, err := DataURI(req.Audio, req.FileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The ceiling is on the whole string as sent — "encoded string size must
|
||||
// not exceed 10 MB" — so the data: prefix counts too.
|
||||
if limit := c.cfg.Settings.MaxEncodedBytes; limit > 0 && len(uri) > limit {
|
||||
return nil, fmt.Errorf(
|
||||
"the audio is %.1f MB and %.1f MB once base64-encoded; this model accepts at most %.1f MB encoded",
|
||||
float64(len(req.Audio))/(1<<20), float64(len(uri))/(1<<20), float64(limit)/(1<<20))
|
||||
}
|
||||
body := map[string]any{
|
||||
"model": c.cfg.Endpoint.Model,
|
||||
"messages": []any{map[string]any{
|
||||
"role": "user",
|
||||
"content": []any{map[string]any{
|
||||
"type": "input_audio",
|
||||
"input_audio": map[string]any{"data": uri},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageASROptions {
|
||||
body["asr_options"] = map[string]any{"language": req.Language}
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
// Content is a pointer for the same reason as the multipart
|
||||
// protocol's text: an absent transcript is an error, an empty one
|
||||
// is silence.
|
||||
Content *string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
// Usage.Seconds is the audio length, which Alibaba reports.
|
||||
Usage struct {
|
||||
Seconds float64 `json:"seconds"`
|
||||
} `json:"usage"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// Transcribe sends one audio file.
|
||||
func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {
|
||||
body, err := c.BuildRequestBody(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded response
|
||||
err = c.cfg.Endpoint.PostJSONWithRetry(ctx, c.url(), body, &decoded, c.cfg.Retry, "transcription")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if decoded.Error != nil && decoded.Error.Message != "" {
|
||||
return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)
|
||||
}
|
||||
if len(decoded.Choices) == 0 || decoded.Choices[0].Message.Content == nil {
|
||||
return nil, fmt.Errorf("transcription reply carries no message content")
|
||||
}
|
||||
return &api.Transcription{
|
||||
Text: strings.TrimSpace(*decoded.Choices[0].Message.Content),
|
||||
Duration: decoded.Usage.Seconds,
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
package openaichataudio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// qwenASRResponse is the non-streaming response example from Alibaba's
|
||||
// Qwen-ASR reference, copied from the page:
|
||||
// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference
|
||||
const qwenASRResponse = `{
|
||||
"choices": [{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"annotations": [{"emotion": "neutral", "language": "zh", "type": "audio_info"}],
|
||||
"content": "欢迎使用阿里云。",
|
||||
"role": "assistant"
|
||||
}
|
||||
}],
|
||||
"created": 1767683986,
|
||||
"id": "chatcmpl-487abe5f-d4f2-9363-a877-xxxxxxx",
|
||||
"model": "qwen3-asr-flash",
|
||||
"object": "chat.completion",
|
||||
"usage": {"completion_tokens": 12, "prompt_tokens": 42, "total_tokens": 54}
|
||||
}`
|
||||
|
||||
// withSeconds is a reply carrying the usage.seconds that the reference's
|
||||
// schema documents ("音频时长(秒)") but its example omits.
|
||||
const withSeconds = `{"choices":[{"message":{"role":"assistant","content":"x"}}],"usage":{"seconds":3}}`
|
||||
|
||||
func newClient(url, model string) *Client {
|
||||
return newClientWith(url, model, catalog.TranscriptionsSettings{})
|
||||
}
|
||||
|
||||
func newClientWith(url, model string, settings catalog.TranscriptionsSettings) *Client {
|
||||
return New(Config{
|
||||
Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},
|
||||
Settings: settings,
|
||||
})
|
||||
}
|
||||
|
||||
// The request is the one both references show: a single user message whose
|
||||
// only part is the audio as a data URI, no instruction, no stream flag.
|
||||
func TestRequestBodyMatchesTheReferences(t *testing.T) {
|
||||
body, err := newClient("https://example.invalid/v1", "qwen3-asr-flash").
|
||||
BuildRequestBody(api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "clip.MP3"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": "qwen3-asr-flash",
|
||||
"messages": []any{map[string]any{
|
||||
"role": "user",
|
||||
"content": []any{map[string]any{
|
||||
"type": "input_audio",
|
||||
"input_audio": map[string]any{"data": "data:audio/mpeg;base64,SUQz"},
|
||||
}},
|
||||
}},
|
||||
}, body)
|
||||
}
|
||||
|
||||
func TestDataURIMediaTypeFollowsTheExtension(t *testing.T) {
|
||||
uri, err := DataURI([]byte("RIFF"), "a.wav")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "data:audio/wav;base64,UklGRg==", uri)
|
||||
|
||||
// No audio type can be named for it, and the references show one.
|
||||
_, err = DataURI([]byte("x"), "noext")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// Both references cap the string as sent, so the data: prefix counts. A file
|
||||
// sized to exactly the encoded budget without the prefix goes over it.
|
||||
func TestEncodedCeilingCountsTheWholeDataURI(t *testing.T) {
|
||||
const limit = 1024
|
||||
c := newClientWith("https://example.invalid", "m", catalog.TranscriptionsSettings{MaxEncodedBytes: limit})
|
||||
audio := make([]byte, limit*3/4) // base64 of this is exactly limit bytes
|
||||
_, err := c.BuildRequestBody(api.TranscriptionRequest{Audio: audio, FileName: "a.wav"})
|
||||
require.Error(t, err)
|
||||
|
||||
prefix := len("data:audio/wav;base64,")
|
||||
fits := make([]byte, (limit-prefix)/4*3)
|
||||
_, err = c.BuildRequestBody(api.TranscriptionRequest{Audio: fits, FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestLanguageGoesInASROptionsWhereDeclared(t *testing.T) {
|
||||
req := api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav", Language: "zh"}
|
||||
|
||||
body, err := newClient("https://example.invalid", "m").BuildRequestBody(req)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, body, "asr_options", "undeclared, so not sent")
|
||||
|
||||
body, err = newClientWith("https://example.invalid", "m",
|
||||
catalog.TranscriptionsSettings{LanguageParam: catalog.LanguageASROptions}).BuildRequestBody(req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"language": "zh"}, body["asr_options"])
|
||||
}
|
||||
|
||||
func TestDecodesTheDocumentedResponse(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
var path string
|
||||
var sent map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&sent)
|
||||
_, _ = w.Write([]byte(qwenASRResponse))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
out, err := newClient(server.URL+"/compatible-mode/v1", "qwen3-asr-flash").
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/compatible-mode/v1/chat/completions", path)
|
||||
assert.Equal(t, &api.Transcription{Text: "欢迎使用阿里云。"}, out)
|
||||
}
|
||||
|
||||
// A reply with no assistant content is not silence.
|
||||
func TestReplyWithoutContentIsAnError(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"choices":[]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := newClient(server.URL, "m").
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestDurationFromUsageSeconds(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(withSeconds))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
out, err := newClient(server.URL, "m").
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, float64(3), out.Duration)
|
||||
}
|
||||
|
||||
func TestEmptyContentIsSilence(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":""}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
out, err := newClient(server.URL, "m").
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", out.Text)
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
// Package openaitranscriptions implements the speech-to-text shape OpenAI
|
||||
// defined and the compatible servers copied: a multipart POST to
|
||||
// {base}/audio/transcriptions carrying file and model, answering {text}, or
|
||||
// {text, segments} when response_format is verbose_json.
|
||||
//
|
||||
// response_format is the one optional field that matters, and it differs by
|
||||
// model: gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json,
|
||||
// whisper-1 also serves verbose_json, SiliconFlow documents no such field at
|
||||
// all. json is the default everywhere it is documented, so nothing is sent
|
||||
// unless catalog.TranscriptionsSettings names a format.
|
||||
//
|
||||
// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create
|
||||
package openaitranscriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
)
|
||||
|
||||
// Config is everything the client needs, already resolved by the catalog.
|
||||
type Config struct {
|
||||
Endpoint api.Endpoint
|
||||
Settings catalog.TranscriptionsSettings
|
||||
Retry api.RetryPolicy
|
||||
}
|
||||
|
||||
// Client talks the OpenAI transcription shape to one endpoint.
|
||||
type Client struct {
|
||||
cfg Config
|
||||
}
|
||||
|
||||
// New builds a client.
|
||||
func New(cfg Config) *Client { return &Client{cfg: cfg} }
|
||||
|
||||
func (c *Client) url() string {
|
||||
if c.cfg.Endpoint.URL != "" {
|
||||
return c.cfg.Endpoint.Resolve("")
|
||||
}
|
||||
path := c.cfg.Settings.Path
|
||||
if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
|
||||
path = ""
|
||||
}
|
||||
return c.cfg.Endpoint.Resolve(path)
|
||||
}
|
||||
|
||||
// FormFields is the golden-test entry point: the text parts of the form, in
|
||||
// the order they are written.
|
||||
func (c *Client) FormFields(language string) []api.FormField {
|
||||
fields := []api.FormField{{Name: "model", Value: c.cfg.Endpoint.Model}}
|
||||
if format := c.cfg.Settings.ResponseFormat; format != "" {
|
||||
fields = append(fields, api.FormField{Name: "response_format", Value: format})
|
||||
}
|
||||
if language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageForm {
|
||||
fields = append(fields, api.FormField{Name: "language", Value: language})
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
type response struct {
|
||||
// Text is a pointer because its absence is an error, not silence. A
|
||||
// transcript of silent audio is a present, empty string; a reply without
|
||||
// the field is some other document. vox-box (GPUStack's audio backend)
|
||||
// returns its HTTPException from the route instead of raising it, so a
|
||||
// failed transcription can arrive as a 2xx whose body is the exception.
|
||||
Text *string `json:"text"`
|
||||
Segments []struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
} `json:"segments"`
|
||||
// Duration is top-level in OpenAI's verbose_json and in MiniMax's
|
||||
// default json; usage.seconds is where OpenAI, OpenRouter and Requesty
|
||||
// report it for models billed by duration.
|
||||
Duration float64 `json:"duration"`
|
||||
Usage struct {
|
||||
Seconds float64 `json:"seconds"`
|
||||
} `json:"usage"`
|
||||
Detail any `json:"detail"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// Transcribe sends one audio file.
|
||||
func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {
|
||||
body, contentType, err := api.EncodeMultipart(c.FormFields(req.Language), api.FormFile{
|
||||
Field: "file", FileName: req.FileName, Data: req.Audio,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpoint := c.cfg.Endpoint
|
||||
if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageHeader {
|
||||
headers := make(map[string]string, len(endpoint.Headers)+1)
|
||||
for k, v := range endpoint.Headers {
|
||||
headers[k] = v
|
||||
}
|
||||
headers["language"] = req.Language
|
||||
endpoint.Headers = headers
|
||||
}
|
||||
var decoded response
|
||||
err = endpoint.PostMultipartWithRetry(
|
||||
ctx, c.url(), body, contentType, &decoded, c.cfg.Retry, "transcription",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if decoded.Error != nil && decoded.Error.Message != "" {
|
||||
return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)
|
||||
}
|
||||
if decoded.Text == nil {
|
||||
if decoded.Detail != nil {
|
||||
return nil, fmt.Errorf("transcription reply carries no text: %v", decoded.Detail)
|
||||
}
|
||||
return nil, fmt.Errorf("transcription reply carries no text")
|
||||
}
|
||||
out := &api.Transcription{Text: strings.TrimSpace(*decoded.Text), Duration: decoded.Duration}
|
||||
if out.Duration == 0 {
|
||||
out.Duration = decoded.Usage.Seconds
|
||||
}
|
||||
for _, s := range decoded.Segments {
|
||||
out.Segments = append(out.Segments, api.TranscriptionSegment{
|
||||
Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
package openaitranscriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// jsonResponse is the json example from the reference, copied from the page:
|
||||
// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create
|
||||
const jsonResponse = `{
|
||||
"text": "text",
|
||||
"languages": [{"code": "code"}],
|
||||
"logprobs": [{"token": "token", "bytes": [0], "logprob": 0}],
|
||||
"usage": {
|
||||
"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "type": "tokens",
|
||||
"input_token_details": {"audio_tokens": 0, "text_tokens": 0}
|
||||
}
|
||||
}`
|
||||
|
||||
// verboseResponse is the "Segment timestamps" example from the same page,
|
||||
// cut to its first segment where the page elides the rest with "...".
|
||||
const verboseResponse = `{
|
||||
"task": "transcribe",
|
||||
"language": "english",
|
||||
"duration": 8.470000267028809,
|
||||
"text": "The beach was a popular spot on a hot summer day. ` +
|
||||
`People were swimming in the ocean, building sandcastles, and playing beach volleyball.",
|
||||
"segments": [
|
||||
{
|
||||
"id": 0,
|
||||
"seek": 0,
|
||||
"start": 0.0,
|
||||
"end": 3.319999933242798,
|
||||
"text": " The beach was a popular spot on a hot summer day.",
|
||||
"tokens": [50364, 440, 7534, 390, 257, 3743, 4008, 322, 257, 2368, 4266, 786, 13, 50530],
|
||||
"temperature": 0.0,
|
||||
"avg_logprob": -0.2860786020755768,
|
||||
"compression_ratio": 1.2363636493682861,
|
||||
"no_speech_prob": 0.00985979475080967
|
||||
}
|
||||
],
|
||||
"usage": {"type": "duration", "seconds": 9}
|
||||
}`
|
||||
|
||||
type captured struct {
|
||||
path string
|
||||
auth string
|
||||
language string
|
||||
fields map[string]string
|
||||
fileName string
|
||||
fileType string
|
||||
file []byte
|
||||
}
|
||||
|
||||
func serve(t *testing.T, status int, reply string) (*httptest.Server, *captured) {
|
||||
t.Helper()
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
got := &captured{fields: map[string]string{}}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got.path = r.URL.Path
|
||||
got.auth = r.Header.Get("Authorization")
|
||||
got.language = r.Header.Get("language")
|
||||
_, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
require.NoError(t, err)
|
||||
reader := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := reader.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err)
|
||||
data, _ := io.ReadAll(part)
|
||||
if part.FormName() == "file" {
|
||||
got.fileName, got.fileType, got.file = part.FileName(), part.Header.Get("Content-Type"), data
|
||||
continue
|
||||
}
|
||||
got.fields[part.FormName()] = string(data)
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write([]byte(reply))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server, got
|
||||
}
|
||||
|
||||
func newClient(url, model string, settings catalog.TranscriptionsSettings) *Client {
|
||||
if settings.Path == "" {
|
||||
settings.Path = catalog.DefaultTranscriptions().Path
|
||||
}
|
||||
return New(Config{
|
||||
Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},
|
||||
Settings: settings,
|
||||
})
|
||||
}
|
||||
|
||||
// The baseline form is file and model. gpt-4o-transcribe accepts only json,
|
||||
// which is the default, so no response_format is sent.
|
||||
func TestBaselineFormIsFileAndModel(t *testing.T) {
|
||||
server, got := serve(t, http.StatusOK, jsonResponse)
|
||||
out, err := newClient(server.URL+"/v1", "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "meeting.wav"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
||||
assert.Equal(t, "Bearer k", got.auth)
|
||||
assert.Equal(t, map[string]string{"model": "gpt-4o-transcribe"}, got.fields)
|
||||
assert.Equal(t, "meeting.wav", got.fileName, "the extension is how the server identifies the format")
|
||||
assert.Empty(t, got.fileType, "a guessed type a server does not list is rejected outright")
|
||||
assert.Equal(t, []byte("RIFF"), got.file)
|
||||
assert.Equal(t, &api.Transcription{Text: "text"}, out)
|
||||
}
|
||||
|
||||
// The language hint goes where the vendor documents it: a form field on
|
||||
// OpenAI's shape, a request header on MiniMax's, nowhere otherwise.
|
||||
func TestLanguageGoesWhereTheVendorDeclares(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
param string
|
||||
wantField string
|
||||
wantHeader string
|
||||
}{
|
||||
{param: "", wantField: "", wantHeader: ""},
|
||||
{param: catalog.LanguageForm, wantField: "zh", wantHeader: ""},
|
||||
{param: catalog.LanguageHeader, wantField: "", wantHeader: "zh"},
|
||||
} {
|
||||
t.Run("param="+tc.param, func(t *testing.T) {
|
||||
server, got := serve(t, http.StatusOK, jsonResponse)
|
||||
_, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{LanguageParam: tc.param}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{
|
||||
Audio: []byte("x"), FileName: "a.wav", Language: "zh",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantField, got.fields["language"])
|
||||
assert.Equal(t, tc.wantHeader, got.language)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The audio length is read from wherever the reply states it, so usage can
|
||||
// be reported without asking for segments.
|
||||
func TestDurationIsReadFromTheReply(t *testing.T) {
|
||||
for name, reply := range map[string]string{
|
||||
// MiniMax's default json, from its reference.
|
||||
"top-level duration": `{"text": "x", "duration": 26.325, "trace_id": "021785229015510a2c883cf675b9804d"}`,
|
||||
// OpenRouter's reference example.
|
||||
"usage.seconds": `{"text": "x", "usage": {"cost": 0.000508, "input_tokens": 83, "output_tokens": 30, ` +
|
||||
`"seconds": 9.2, "total_tokens": 113}}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
server, _ := serve(t, http.StatusOK, reply)
|
||||
out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, out.Duration)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerboseJSONCarriesSegments(t *testing.T) {
|
||||
server, got := serve(t, http.StatusOK, verboseResponse)
|
||||
settings := catalog.TranscriptionsSettings{ResponseFormat: "verbose_json"}
|
||||
out, err := newClient(server.URL+"/v1", "whisper-1", settings).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "a.mp3"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "verbose_json", got.fields["response_format"])
|
||||
require.Len(t, out.Segments, 1)
|
||||
assert.Equal(t, api.TranscriptionSegment{
|
||||
Start: 0, End: 3.319999933242798, Text: "The beach was a popular spot on a hot summer day.",
|
||||
}, out.Segments[0])
|
||||
}
|
||||
|
||||
// Silence is an empty transcript, and that is a valid answer.
|
||||
func TestEmptyTextIsSilence(t *testing.T) {
|
||||
server, _ := serve(t, http.StatusOK, `{"text": ""}`)
|
||||
out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", out.Text)
|
||||
}
|
||||
|
||||
// A reply without text is not silence. vox-box returns its HTTPException
|
||||
// from the route rather than raising it, so a failure can arrive as a 2xx
|
||||
// body shaped like this; reading it as an empty transcript would store "no
|
||||
// speech detected" for a file that was never transcribed.
|
||||
func TestReplyWithoutTextIsAnError(t *testing.T) {
|
||||
server, _ := serve(t, http.StatusOK,
|
||||
`{"status_code": 500, "detail": "Failed to transcribe audio, boom", "headers": null}`)
|
||||
_, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "Failed to transcribe audio")
|
||||
}
|
||||
|
||||
func TestSurfacesTheVendorErrorBody(t *testing.T) {
|
||||
// A stand-in rejection in OpenAI's error envelope.
|
||||
server, _ := serve(t, http.StatusBadRequest, `{"error":{"message":"stand-in rejection"}}`)
|
||||
_, err := newClient(server.URL, "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "stand-in rejection")
|
||||
}
|
||||
|
||||
func TestDoesNotDoubleThePath(t *testing.T) {
|
||||
server, got := serve(t, http.StatusOK, jsonResponse)
|
||||
_, err := newClient(server.URL+"/v1/audio/transcriptions", "m", catalog.TranscriptionsSettings{}).
|
||||
Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/v1/audio/transcriptions", got.path)
|
||||
}
|
||||
|
|
@ -49,12 +49,23 @@ func (p RetryPolicy) delay(attempt int) time.Duration {
|
|||
// Here the last error is captured in the enclosing scope on purpose and the
|
||||
// function cannot fall through without it.
|
||||
//
|
||||
// Only transport failures are retried. A non-2xx reply is the vendor
|
||||
// answering, and repeating a rejected request neither fixes it nor tells the
|
||||
// operator anything new.
|
||||
// Only transport failures are retried, including a reply cut off while it
|
||||
// was being read. A non-2xx reply is the vendor answering, and repeating a
|
||||
// rejected request neither fixes it nor tells the operator anything new; a
|
||||
// reply that arrived whole but does not decode is the same answer every time,
|
||||
// and sending it again would bill the request twice.
|
||||
func (e Endpoint) PostJSONWithRetry(
|
||||
ctx context.Context, url string, body, out any, policy RetryPolicy, label string,
|
||||
) error {
|
||||
return withRetry(ctx, policy, label, func() error {
|
||||
return e.PostJSON(ctx, url, body, out)
|
||||
})
|
||||
}
|
||||
|
||||
// withRetry runs send until it succeeds, fails with anything other than a
|
||||
// TransportError, or the budget runs out. The last error lives in the
|
||||
// enclosing scope and the loop cannot fall through without it.
|
||||
func withRetry(ctx context.Context, policy RetryPolicy, label string, send func() error) error {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= policy.MaxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
|
|
@ -67,12 +78,12 @@ func (e Endpoint) PostJSONWithRetry(
|
|||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
err := e.PostJSON(ctx, url, body, out)
|
||||
err := send()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var httpErr *HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
var transportErr *TransportError
|
||||
if !errors.As(err, &transportErr) {
|
||||
return err
|
||||
}
|
||||
lastErr = err
|
||||
|
|
|
|||
|
|
@ -78,6 +78,25 @@ func TestPostJSONWithRetryDoesNotRetryAVendorRejection(t *testing.T) {
|
|||
assert.Equal(t, 1, attempts, "a rejection must not be repeated")
|
||||
}
|
||||
|
||||
// A reply that arrived but does not decode is not a transport failure: the
|
||||
// vendor did the work, and sending the request again bills it again.
|
||||
func TestPostJSONWithRetryDoesNotRetryAnUndecodableReply(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
attempts := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
attempts++
|
||||
_, _ = w.Write([]byte(`not json`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
var out map[string]any
|
||||
err := Endpoint{}.PostJSONWithRetry(
|
||||
context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",
|
||||
)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, attempts)
|
||||
}
|
||||
|
||||
func TestPostJSONWithRetryStopsWhenTheContextIsDone(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
||||
|
|
@ -101,3 +120,54 @@ func TestRetryPolicyBackoffDoublesAndIsCapped(t *testing.T) {
|
|||
assert.Equal(t, 4*time.Second, p.delay(4), "capped")
|
||||
assert.Zero(t, RetryPolicy{MaxRetries: 2}.delay(1), "no base delay means no waiting")
|
||||
}
|
||||
|
||||
// A reply cut off mid-body never arrived, so it is sent again.
|
||||
func TestPostJSONWithRetryRetriesAReplyCutOffWhileReading(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
attempts := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
// Promise more than is sent, then hang up.
|
||||
w.Header().Set("Content-Length", "100")
|
||||
_, _ = w.Write([]byte(`{"a":`))
|
||||
w.(http.Flusher).Flush() // the status and half a body are on the wire
|
||||
hj, ok := w.(http.Hijacker)
|
||||
require.True(t, ok)
|
||||
conn, _, err := hj.Hijack()
|
||||
require.NoError(t, err)
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"a":1}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
var out map[string]any
|
||||
err := Endpoint{}.PostJSONWithRetry(
|
||||
context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, attempts)
|
||||
assert.Equal(t, float64(1), out["a"])
|
||||
}
|
||||
|
||||
// The phase that failed is named, so the log says whether the request or the
|
||||
// reply was lost.
|
||||
func TestReadFailureIsReportedAsSuch(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "100")
|
||||
_, _ = w.Write([]byte(`{"a":`))
|
||||
w.(http.Flusher).Flush()
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
require.NoError(t, err)
|
||||
_ = conn.Close()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
err := Endpoint{}.PostJSON(context.Background(), server.URL, map[string]any{}, &map[string]any{})
|
||||
var transportErr *TransportError
|
||||
require.ErrorAs(t, err, &transportErr)
|
||||
assert.Equal(t, "read response", transportErr.Op)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
package api
|
||||
|
||||
import "context"
|
||||
|
||||
// TranscriptionAPI names a speech-to-text wire protocol. Like RerankAPI and
|
||||
// EmbeddingAPI it is its own type, so no other modality's value validates.
|
||||
type TranscriptionAPI string
|
||||
|
||||
// The transcription protocols WeKnora speaks.
|
||||
const (
|
||||
// TranscriptionOpenAI is POST {base}/audio/transcriptions as a multipart
|
||||
// form carrying file and model, answering {text} — or {text, segments}
|
||||
// when verbose_json is asked for.
|
||||
TranscriptionOpenAI TranscriptionAPI = "openai-transcriptions"
|
||||
// TranscriptionChatAudio is a dedicated speech-recognition model behind
|
||||
// POST {base}/chat/completions: the audio goes in as a base64 data URI in
|
||||
// an input_audio content part and the transcript comes back as the
|
||||
// assistant message. Alibaba's qwen3-asr-flash and Xiaomi's mimo-v2.5-asr
|
||||
// take no instruction; the model only transcribes.
|
||||
TranscriptionChatAudio TranscriptionAPI = "openai-chat-audio"
|
||||
)
|
||||
|
||||
// Known reports whether the value names a protocol this build implements.
|
||||
func (a TranscriptionAPI) Known() bool {
|
||||
return a == TranscriptionOpenAI || a == TranscriptionChatAudio
|
||||
}
|
||||
|
||||
// TranscriptionSegment is one timed stretch of a transcript.
|
||||
type TranscriptionSegment struct {
|
||||
Start float64
|
||||
End float64
|
||||
Text string
|
||||
}
|
||||
|
||||
// Transcription is what a protocol client returns.
|
||||
type Transcription struct {
|
||||
Text string
|
||||
Segments []TranscriptionSegment
|
||||
// Duration is the audio length in seconds when the reply states it —
|
||||
// OpenAI-shaped duration or usage.seconds — and 0 otherwise.
|
||||
Duration float64
|
||||
}
|
||||
|
||||
// TranscriptionRequest is one file to transcribe.
|
||||
type TranscriptionRequest struct {
|
||||
Audio []byte
|
||||
FileName string
|
||||
// Language is the operator's hint, empty for auto-detection. Where it is
|
||||
// sent, and whether at all, is the vendor's declaration.
|
||||
Language string
|
||||
}
|
||||
|
||||
// Transcriber turns one audio file into text.
|
||||
type Transcriber interface {
|
||||
Transcribe(ctx context.Context, req TranscriptionRequest) (*Transcription, error)
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package asr
|
|||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
|
@ -17,6 +18,27 @@ type Segment struct {
|
|||
type TranscriptionResult struct {
|
||||
Text string `json:"text"`
|
||||
Segments []Segment `json:"segments,omitempty"`
|
||||
// Duration is the audio length in seconds when the vendor reports it,
|
||||
// which several do without segments; 0 when unknown.
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
}
|
||||
|
||||
type languageKey struct{}
|
||||
|
||||
// WithLanguage attaches the operator's language hint (for example "zh") to
|
||||
// a transcription. Empty and "auto" mean auto-detection, which is every
|
||||
// vendor's default, and send nothing.
|
||||
func WithLanguage(ctx context.Context, language string) context.Context {
|
||||
return context.WithValue(ctx, languageKey{}, language)
|
||||
}
|
||||
|
||||
func languageFrom(ctx context.Context) string {
|
||||
language, _ := ctx.Value(languageKey{}).(string)
|
||||
language = strings.TrimSpace(language)
|
||||
if strings.EqualFold(language, "auto") {
|
||||
return ""
|
||||
}
|
||||
return language
|
||||
}
|
||||
|
||||
// ASR defines the interface for Automatic Speech Recognition model operations.
|
||||
|
|
@ -35,7 +57,10 @@ type Config struct {
|
|||
ModelName string
|
||||
APIKey string
|
||||
ModelID string
|
||||
Language string // optional: specify language for transcription
|
||||
// Provider is the vendor id stored on the row; empty detects it from
|
||||
// BaseURL.
|
||||
Provider string
|
||||
ExtraConfig map[string]string
|
||||
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
CustomHeaders map[string]string
|
||||
}
|
||||
|
|
@ -53,13 +78,14 @@ func ConfigFromModel(m *types.Model) *Config {
|
|||
BaseURL: m.Parameters.BaseURL,
|
||||
ModelName: m.Name,
|
||||
Source: m.Source,
|
||||
Provider: m.Parameters.Provider,
|
||||
ExtraConfig: m.Parameters.ExtraConfig,
|
||||
CustomHeaders: m.Parameters.CustomHeaders,
|
||||
}
|
||||
}
|
||||
|
||||
// NewASR creates an ASR instance based on the provided configuration.
|
||||
// All ASR vendors use the OpenAI-compatible /v1/audio/transcriptions API.
|
||||
func NewASR(config *Config) (ASR, error) {
|
||||
a, err := NewOpenAIASR(config)
|
||||
a, err := newASR(config)
|
||||
return wrapASRLangfuse(a, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ func TestConfigFromModel(t *testing.T) {
|
|||
Parameters: types.ModelParameters{
|
||||
BaseURL: "https://api.example.com/v1",
|
||||
APIKey: "sk",
|
||||
Provider: "siliconflow",
|
||||
ExtraConfig: map[string]string{"remote_model_name": "r"},
|
||||
CustomHeaders: map[string]string{"X": "y"},
|
||||
},
|
||||
}
|
||||
|
|
@ -24,6 +26,11 @@ func TestConfigFromModel(t *testing.T) {
|
|||
if cfg.BaseURL != "https://api.example.com/v1" || cfg.APIKey != "sk" {
|
||||
t.Errorf("connection fields mismatch: %+v", cfg)
|
||||
}
|
||||
// The vendor decides the wire format; without it every row resolves to
|
||||
// the generic runtime.
|
||||
if cfg.Provider != "siliconflow" || cfg.ExtraConfig["remote_model_name"] != "r" {
|
||||
t.Errorf("vendor fields not propagated: %+v", cfg)
|
||||
}
|
||||
if cfg.CustomHeaders["X"] != "y" {
|
||||
t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ import (
|
|||
|
||||
// langfuseASR wraps an ASR implementation and reports each Transcribe call
|
||||
// as a Langfuse generation observation. Audio bytes are not uploaded — we
|
||||
// record file name, audio size and duration (derived from segment end times)
|
||||
// record file name, audio size and duration (as the vendor reports it, or
|
||||
// from segment end times)
|
||||
// so usage can be billed by second/minute, which is how most ASR providers
|
||||
// price their services.
|
||||
type langfuseASR struct {
|
||||
|
|
@ -44,8 +45,13 @@ func (l *langfuseASR) Transcribe(ctx context.Context, audioBytes []byte, fileNam
|
|||
if result != nil {
|
||||
output["text"] = result.Text
|
||||
output["segment_count"] = len(result.Segments)
|
||||
if n := len(result.Segments); n > 0 {
|
||||
// The vendor's own figure when it gives one; otherwise the end of
|
||||
// the last segment, which only verbose replies carry.
|
||||
duration = result.Duration
|
||||
if n := len(result.Segments); duration == 0 && n > 0 {
|
||||
duration = result.Segments[n-1].End
|
||||
}
|
||||
if duration > 0 {
|
||||
output["duration_seconds"] = duration
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,150 +0,0 @@
|
|||
package asr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
openai "github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
const (
|
||||
asrDefaultTimeout = 300 * time.Second // audio transcription can be slow
|
||||
)
|
||||
|
||||
// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API.
|
||||
type OpenAIASR struct {
|
||||
modelName string
|
||||
modelID string
|
||||
client *openai.Client
|
||||
baseURL string
|
||||
language string
|
||||
}
|
||||
|
||||
// NewOpenAIASR creates an OpenAI-compatible ASR instance.
|
||||
func NewOpenAIASR(config *Config) (*OpenAIASR, error) {
|
||||
if err := validateASRBaseURL(config.BaseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
apiCfg := openai.DefaultConfig(config.APIKey)
|
||||
if config.BaseURL != "" {
|
||||
apiCfg.BaseURL = config.BaseURL
|
||||
}
|
||||
httpClient := newASRHTTPClient(asrDefaultTimeout)
|
||||
|
||||
// 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers)
|
||||
if len(config.CustomHeaders) > 0 {
|
||||
apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders)
|
||||
} else {
|
||||
apiCfg.HTTPClient = httpClient
|
||||
}
|
||||
|
||||
return &OpenAIASR{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
client: openai.NewClientWithConfig(apiCfg),
|
||||
baseURL: config.BaseURL,
|
||||
language: config.Language,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.
|
||||
func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error) {
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, fmt.Errorf("audio bytes are empty")
|
||||
}
|
||||
|
||||
// Ensure fileName has an extension for proper MIME type detection
|
||||
if fileName == "" {
|
||||
fileName = "audio.mp3"
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s",
|
||||
s.modelName, s.baseURL, len(audioBytes), fileName)
|
||||
|
||||
req := openai.AudioRequest{
|
||||
Model: s.modelName,
|
||||
FilePath: fileName,
|
||||
Reader: bytes.NewReader(audioBytes),
|
||||
Format: openai.AudioResponseFormatVerboseJSON,
|
||||
}
|
||||
|
||||
if s.language != "" {
|
||||
req.Language = s.language
|
||||
}
|
||||
|
||||
resp, err := s.client.CreateTranscription(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ASR transcription request failed: %w", err)
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcription response: %w", err)
|
||||
}
|
||||
logger.Debugf(ctx, "[ASR] Transcription response: %s", string(jsonData))
|
||||
|
||||
text := strings.TrimSpace(resp.Text)
|
||||
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))
|
||||
|
||||
var segments []Segment
|
||||
for _, s := range resp.Segments {
|
||||
segments = append(segments, Segment{
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: strings.TrimSpace(s.Text),
|
||||
})
|
||||
}
|
||||
|
||||
return &TranscriptionResult{
|
||||
Text: text,
|
||||
Segments: segments,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *OpenAIASR) GetModelName() string { return s.modelName }
|
||||
func (s *OpenAIASR) GetModelID() string { return s.modelID }
|
||||
|
||||
// DetectAudioFormat returns a file extension hint for the given audio bytes.
|
||||
func DetectAudioFormat(data []byte, fileName string) string {
|
||||
if fileName != "" {
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if ext != "" {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
// Try to detect from magic bytes
|
||||
if len(data) >= 4 {
|
||||
// MP3: starts with ID3 or 0xFF 0xFB
|
||||
if data[0] == 'I' && data[1] == 'D' && data[2] == '3' {
|
||||
return ".mp3"
|
||||
}
|
||||
if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {
|
||||
return ".mp3"
|
||||
}
|
||||
// FLAC: starts with "fLaC"
|
||||
if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' {
|
||||
return ".flac"
|
||||
}
|
||||
// OGG: starts with "OggS"
|
||||
if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' {
|
||||
return ".ogg"
|
||||
}
|
||||
// WAV: starts with "RIFF"
|
||||
if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' {
|
||||
return ".wav"
|
||||
}
|
||||
}
|
||||
// M4A: check for ftyp box
|
||||
if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' {
|
||||
return ".m4a"
|
||||
}
|
||||
return ".mp3" // default
|
||||
}
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/api/openaichataudio"
|
||||
"github.com/Tencent/WeKnora/internal/models/api/openaitranscriptions"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
// catalog.Resolve answers from the vendor catalog, which is empty until
|
||||
// the vendor packages have run their init.
|
||||
_ "github.com/Tencent/WeKnora/internal/models/vendors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// retryPolicy is the transport-error retry budget, which is none. The
|
||||
// knowledge pipeline retries a failed transcription as a whole task, and a
|
||||
// client-level retry on top would turn one hung 300-second upload into
|
||||
// several.
|
||||
var retryPolicy = func() api.RetryPolicy { return api.RetryPolicy{} }
|
||||
|
||||
// newASR resolves the catalog and returns the protocol client for the
|
||||
// configured model. It mirrors rerank.newReranker and
|
||||
// embedding.newRemoteEmbedder: the vendor's facts decide the protocol, the
|
||||
// URL and the credential, and this function knows no vendor names.
|
||||
func newASR(config *Config) (ASR, error) {
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("asr config is nil")
|
||||
}
|
||||
if strings.TrimSpace(config.ModelName) == "" {
|
||||
return nil, fmt.Errorf("model name is required")
|
||||
}
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: config.Provider,
|
||||
Model: config.ModelName,
|
||||
BaseURL: config.BaseURL,
|
||||
ModelType: types.ModelTypeASR,
|
||||
Extra: config.ExtraConfig,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A vendor that does not declare speech recognition has not been checked
|
||||
// for it, and its Endpoint hook may still compute a path — Azure's does.
|
||||
// A row that named the vendor is refused; a row that named none and was
|
||||
// matched by its URL is what the pre-catalog client served: an
|
||||
// OpenAI-compatible endpoint at that URL.
|
||||
if !resolved.Vendor.SupportsType(types.ModelTypeASR) {
|
||||
if strings.TrimSpace(config.Provider) != "" {
|
||||
return nil, fmt.Errorf("%s does not offer speech recognition in this build", resolved.Vendor.Name)
|
||||
}
|
||||
resolved, err = catalog.Resolve(catalog.Ref{
|
||||
Provider: catalog.GenericID,
|
||||
Model: config.ModelName,
|
||||
BaseURL: config.BaseURL,
|
||||
ModelType: types.ModelTypeASR,
|
||||
Extra: config.ExtraConfig,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := validateASRBaseURL(resolved.BaseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vendor := resolved.Vendor
|
||||
creds := catalog.Credentials{APIKey: config.APIKey}
|
||||
if creds.APIKey == "" {
|
||||
creds.APIKey = vendor.DefaultAPIKey
|
||||
}
|
||||
// Vendor headers first so a user header cannot silently replace a vendor
|
||||
// beta flag, matching chat.NewRemoteChat.
|
||||
headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders))
|
||||
for k, v := range vendor.Headers {
|
||||
headers[k] = v
|
||||
}
|
||||
for k, v := range config.CustomHeaders {
|
||||
headers[k] = v
|
||||
}
|
||||
settings := resolved.Transcriptions
|
||||
endpoint := api.Endpoint{
|
||||
BaseURL: resolved.BaseURL,
|
||||
Model: resolved.RemoteModel,
|
||||
ModelID: config.ModelID,
|
||||
Auth: vendor.AuthFunc(vendor.API, creds),
|
||||
Headers: headers,
|
||||
Client: newASRHTTPClient(time.Duration(settings.RequestTimeout) * time.Second),
|
||||
}
|
||||
if vendor.Endpoint != nil {
|
||||
if url, query := vendor.Endpoint(catalog.EndpointRequest{
|
||||
BaseURL: resolved.BaseURL,
|
||||
Model: resolved.RemoteModel,
|
||||
ModelType: types.ModelTypeASR,
|
||||
API: vendor.API,
|
||||
Extra: config.ExtraConfig,
|
||||
}); url != "" {
|
||||
if err := validateASRBaseURL(url); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpoint.URL, endpoint.Query = url, query
|
||||
}
|
||||
}
|
||||
|
||||
var client api.Transcriber
|
||||
switch resolved.TranscriptionAPI {
|
||||
case api.TranscriptionOpenAI:
|
||||
client = openaitranscriptions.New(openaitranscriptions.Config{
|
||||
Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),
|
||||
})
|
||||
case api.TranscriptionChatAudio:
|
||||
client = openaichataudio.New(openaichataudio.Config{
|
||||
Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),
|
||||
})
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported transcription api %q for provider %s",
|
||||
resolved.TranscriptionAPI, vendor.ID)
|
||||
}
|
||||
return &protocolASR{
|
||||
inner: client,
|
||||
settings: settings,
|
||||
vendor: vendor.Name,
|
||||
endpoint: resolved.BaseURL,
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// protocolASR adapts a protocol client to the ASR interface and refuses an
|
||||
// upload the vendor has documented it will not take, before sending it.
|
||||
type protocolASR struct {
|
||||
inner api.Transcriber
|
||||
settings catalog.TranscriptionsSettings
|
||||
vendor string
|
||||
endpoint string
|
||||
modelName string
|
||||
modelID string
|
||||
}
|
||||
|
||||
func (a *protocolASR) GetModelName() string { return a.modelName }
|
||||
func (a *protocolASR) GetModelID() string { return a.modelID }
|
||||
|
||||
func (a *protocolASR) Transcribe(ctx context.Context, audio []byte, fileName string) (*TranscriptionResult, error) {
|
||||
if len(audio) == 0 {
|
||||
return nil, fmt.Errorf("audio bytes are empty")
|
||||
}
|
||||
if limit := a.settings.MaxFileBytes; limit > 0 && len(audio) > limit {
|
||||
return nil, fmt.Errorf("%s transcription: the audio is %.1f MB; %s accepts at most %d MB per file",
|
||||
a.modelName, float64(len(audio))/(1<<20), a.vendor, limit>>20)
|
||||
}
|
||||
// The server identifies the format from the extension.
|
||||
if fileName == "" {
|
||||
fileName = "audio.mp3"
|
||||
}
|
||||
if formats := a.settings.Formats; len(formats) > 0 {
|
||||
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(fileName)), ".")
|
||||
if !slices.Contains(formats, ext) {
|
||||
return nil, fmt.Errorf("%s transcription: %s accepts %s audio, not %q",
|
||||
a.modelName, a.vendor, strings.Join(formats, "/"), fileName)
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "[ASR] transcribing model=%s endpoint=%s size=%d file=%s",
|
||||
a.modelName, a.endpoint, len(audio), fileName)
|
||||
|
||||
out, err := a.inner.Transcribe(ctx, api.TranscriptionRequest{
|
||||
Audio: audio, FileName: fileName, Language: languageFrom(ctx),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ASR transcription request failed: %w", err)
|
||||
}
|
||||
result := &TranscriptionResult{Text: out.Text, Duration: out.Duration}
|
||||
for _, s := range out.Segments {
|
||||
result.Segments = append(result.Segments, Segment{Start: s.Start, End: s.End, Text: s.Text})
|
||||
}
|
||||
logger.Infof(ctx, "[ASR] transcription completed, text length=%d", len(result.Text))
|
||||
return result, nil
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
package asr
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func withASRSSRFWhitelist(t *testing.T, raw string) {
|
||||
t.Helper()
|
||||
t.Setenv("SSRF_WHITELIST", raw)
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
}
|
||||
|
||||
func TestOpenAIASRRejectsInternalBaseURL(t *testing.T) {
|
||||
withASRSSRFWhitelist(t, "")
|
||||
|
||||
_, err := NewOpenAIASR(&Config{
|
||||
BaseURL: "http://169.254.169.254/latest/meta-data/",
|
||||
ModelName: "asr-test",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("NewOpenAIASR returned nil error for blocked internal BaseURL")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,356 @@
|
|||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func allowLoopback(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
}
|
||||
|
||||
type form struct {
|
||||
path string
|
||||
auth string
|
||||
fields map[string]string
|
||||
fileName string
|
||||
language string // the language request header
|
||||
// body is set instead of fields when the request was JSON: the
|
||||
// chat-served recognisers take the audio as a data URI.
|
||||
body map[string]any
|
||||
}
|
||||
|
||||
// upstream records the form it receives and answers the documented json
|
||||
// shape, adding a segment when verbose_json was asked for.
|
||||
func upstream(t *testing.T) (string, *form, *atomic.Int32) {
|
||||
t.Helper()
|
||||
allowLoopback(t)
|
||||
got := &form{fields: map[string]string{}}
|
||||
calls := &atomic.Int32{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
got.path, got.auth, got.language = r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("language")
|
||||
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
require.NoError(t, err)
|
||||
if mediaType == "application/json" {
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&got.body))
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":" hello "}}]}`))
|
||||
return
|
||||
}
|
||||
reader := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := reader.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err)
|
||||
data, _ := io.ReadAll(part)
|
||||
if part.FormName() == "file" {
|
||||
got.fileName = part.FileName()
|
||||
continue
|
||||
}
|
||||
got.fields[part.FormName()] = string(data)
|
||||
}
|
||||
if got.fields["response_format"] == "verbose_json" {
|
||||
_, _ = w.Write([]byte(`{"text":" hello ","segments":[{"start":0,"end":1.5,"text":" hello "}]}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"text":" hello "}`))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server.URL, got, calls
|
||||
}
|
||||
|
||||
// TestTranscriptionWireFormatPerVendor pins the form each ASR vendor is sent.
|
||||
// response_format appears only where the vendor documents it for that model:
|
||||
// the pre-catalog client sent verbose_json to every one of them, which
|
||||
// gpt-4o-transcribe rejects ("the only supported format is json") and which
|
||||
// vox-box's FunASR backend answers with an undecodable bare string.
|
||||
func TestTranscriptionWireFormatPerVendor(t *testing.T) {
|
||||
chatAudio := func(model string) map[string]any {
|
||||
return map[string]any{
|
||||
"model": model,
|
||||
"messages": []any{map[string]any{
|
||||
"role": "user",
|
||||
"content": []any{map[string]any{
|
||||
"type": "input_audio",
|
||||
"input_audio": map[string]any{"data": "data:audio/wav;base64,UklGRg=="},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
}
|
||||
cases := []struct {
|
||||
name, provider, model, base string
|
||||
wantPath string
|
||||
wantFields map[string]string
|
||||
wantBody map[string]any
|
||||
wantSegments int
|
||||
}{
|
||||
{
|
||||
name: "openai whisper-1 serves segments", provider: "openai", model: "whisper-1", base: "/v1",
|
||||
wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "whisper-1", "response_format": "verbose_json"},
|
||||
wantSegments: 1,
|
||||
},
|
||||
{
|
||||
name: "openai gpt-4o-transcribe accepts only json", provider: "openai", model: "gpt-4o-transcribe",
|
||||
base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "gpt-4o-transcribe"},
|
||||
},
|
||||
{
|
||||
name: "openai model not yet in the catalog", provider: "openai", model: "gpt-5-transcribe",
|
||||
base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "gpt-5-transcribe"},
|
||||
},
|
||||
{
|
||||
name: "siliconflow documents file and model only", provider: "siliconflow",
|
||||
model: "FunAudioLLM/SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "FunAudioLLM/SenseVoiceSmall"},
|
||||
},
|
||||
{
|
||||
name: "gpustack stays on json for its FunASR backend", provider: "gpustack",
|
||||
model: "SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "SenseVoiceSmall"},
|
||||
},
|
||||
{
|
||||
name: "generic stays on json", provider: "generic", model: "whisper-large-v3",
|
||||
base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "whisper-large-v3"},
|
||||
},
|
||||
{
|
||||
name: "zhipu glm-asr on the OpenAI shape", provider: "zhipu", model: "glm-asr-2512",
|
||||
base: "/api/paas/v4", wantPath: "/api/paas/v4/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "glm-asr-2512"},
|
||||
},
|
||||
{
|
||||
name: "minimax on its own path", provider: "minimax", model: "asr-1.0",
|
||||
base: "/v1", wantPath: "/v1/speech_to_text",
|
||||
wantFields: map[string]string{"model": "asr-1.0"},
|
||||
},
|
||||
{
|
||||
name: "openrouter", provider: "openrouter", model: "openai/whisper-large-v3",
|
||||
base: "/api/v1", wantPath: "/api/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "openai/whisper-large-v3"},
|
||||
},
|
||||
{
|
||||
name: "requesty whisper-1 serves segments", provider: "requesty", model: "openai/whisper-1",
|
||||
base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "openai/whisper-1", "response_format": "verbose_json"},
|
||||
wantSegments: 1,
|
||||
},
|
||||
{
|
||||
name: "litellm proxy", provider: "litellm", model: "whisper",
|
||||
base: "/v1", wantPath: "/v1/audio/transcriptions",
|
||||
wantFields: map[string]string{"model": "whisper"},
|
||||
},
|
||||
{
|
||||
name: "aliyun qwen3-asr-flash is served on chat", provider: "aliyun", model: "qwen3-asr-flash",
|
||||
base: "/compatible-mode/v1", wantPath: "/compatible-mode/v1/chat/completions",
|
||||
wantBody: chatAudio("qwen3-asr-flash"),
|
||||
},
|
||||
{
|
||||
name: "mimo asr is served on chat", provider: "mimo", model: "mimo-v2.5-asr",
|
||||
base: "/v1", wantPath: "/v1/chat/completions",
|
||||
wantBody: chatAudio("mimo-v2.5-asr"),
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
url, got, _ := upstream(t)
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,
|
||||
ModelName: tc.model, APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
out, err := a.Transcribe(context.Background(), []byte("RIFF"), "meeting.wav")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, tc.wantPath, got.path)
|
||||
assert.Equal(t, "Bearer k", got.auth)
|
||||
if tc.wantBody != nil {
|
||||
assert.Equal(t, tc.wantBody, got.body)
|
||||
} else {
|
||||
assert.Equal(t, tc.wantFields, got.fields)
|
||||
assert.Equal(t, "meeting.wav", got.fileName)
|
||||
}
|
||||
assert.Equal(t, "hello", out.Text)
|
||||
assert.Len(t, out.Segments, tc.wantSegments)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A file over the documented ceiling is refused before it is uploaded: the
|
||||
// vendor would reject it anyway, after the whole upload.
|
||||
func TestOversizedAudioIsRefusedBeforeUpload(t *testing.T) {
|
||||
url, _, calls := upstream(t)
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1",
|
||||
ModelName: "whisper-1", APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = a.Transcribe(context.Background(), make([]byte, 25<<20+1), "long.mp3")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "at most 25 MB")
|
||||
assert.Zero(t, calls.Load(), "nothing should have been sent")
|
||||
|
||||
_, err = a.Transcribe(context.Background(), make([]byte, 25<<20), "exactly.mp3")
|
||||
require.NoError(t, err, "the ceiling itself is allowed")
|
||||
}
|
||||
|
||||
// The chat-served recognisers cap the data URI as sent at 10 MB, prefix
|
||||
// included, so three quarters of 10 MB of audio is already too much.
|
||||
func TestChatServedCeilingCountsTheWholeDataURI(t *testing.T) {
|
||||
url, _, calls := upstream(t)
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "aliyun", BaseURL: url + "/compatible-mode/v1",
|
||||
ModelName: "qwen3-asr-flash", APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = a.Transcribe(context.Background(), make([]byte, 10<<20*3/4), "long.wav")
|
||||
require.Error(t, err)
|
||||
assert.Zero(t, calls.Load())
|
||||
|
||||
prefix := len("data:audio/wav;base64,")
|
||||
_, err = a.Transcribe(context.Background(), make([]byte, (10<<20-prefix)/4*3), "fits.wav")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Only qwen3-asr-flash takes the audio in the request. Alibaba's other
|
||||
// recognition models want a public URL or an asynchronous task, so a row
|
||||
// naming one is refused with the reason rather than sent a chat request.
|
||||
func TestAlibabaRecognitionModelsOtherThanQwenASRFlashAreRefused(t *testing.T) {
|
||||
_, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "aliyun",
|
||||
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", ModelName: "paraformer-v2", APIKey: "k",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "public file URL")
|
||||
}
|
||||
|
||||
// A vendor that does not declare speech recognition is not trusted with it.
|
||||
// Azure's Endpoint hook still knows a transcription path, but nobody has
|
||||
// verified it; a row that names Azure is refused, and a row that names no
|
||||
// vendor gets what the pre-catalog client sent: the OpenAI shape at its URL.
|
||||
func TestVendorsWithoutASRAreNotRoutedThroughTheirHooks(t *testing.T) {
|
||||
_, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "azure_openai",
|
||||
BaseURL: "https://example.openai.azure.com", ModelName: "whisper", APIKey: "k",
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Detection matches URL patterns as substrings, so a path carrying Azure's
|
||||
// host detects as Azure while the request still reaches the test server.
|
||||
url, got, _ := upstream(t)
|
||||
base := url + "/openai.azure.com/v1"
|
||||
require.Equal(t, "azure_openai", catalog.DetectByURL(base))
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, BaseURL: base, ModelName: "whisper", APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = a.Transcribe(context.Background(), []byte("RIFF"), "a.wav")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/openai.azure.com/v1/audio/transcriptions", got.path, "not Azure's /openai/v1 path")
|
||||
assert.Equal(t, "Bearer k", got.auth, "not Azure's api-key header")
|
||||
}
|
||||
|
||||
// A format the vendor does not list is refused before upload.
|
||||
func TestUndocumentedFormatIsRefusedBeforeUpload(t *testing.T) {
|
||||
url, _, calls := upstream(t)
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "zhipu", BaseURL: url + "/api/paas/v4",
|
||||
ModelName: "glm-asr-2512", APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = a.Transcribe(context.Background(), []byte("x"), "memo.m4a")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "wav/mp3")
|
||||
assert.Zero(t, calls.Load())
|
||||
}
|
||||
|
||||
// The knowledge base's language hint reaches each vendor where its reference
|
||||
// puts it, and nowhere for a vendor that documents none.
|
||||
func TestLanguageHintGoesWhereEachVendorDocumentsIt(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, provider, model, base string
|
||||
check func(t *testing.T, got *form)
|
||||
}{
|
||||
{"openai form field", "openai", "whisper-1", "/v1", func(t *testing.T, got *form) {
|
||||
assert.Equal(t, "zh", got.fields["language"])
|
||||
assert.Empty(t, got.language)
|
||||
}},
|
||||
{"minimax header", "minimax", "asr-1.0", "/v1", func(t *testing.T, got *form) {
|
||||
assert.Equal(t, "zh", got.language)
|
||||
assert.NotContains(t, got.fields, "language")
|
||||
}},
|
||||
{"aliyun asr_options", "aliyun", "qwen3-asr-flash", "/compatible-mode/v1", func(t *testing.T, got *form) {
|
||||
assert.Equal(t, map[string]any{"language": "zh"}, got.body["asr_options"])
|
||||
}},
|
||||
{
|
||||
"siliconflow documents none", "siliconflow", "FunAudioLLM/SenseVoiceSmall", "/v1",
|
||||
func(t *testing.T, got *form) {
|
||||
assert.NotContains(t, got.fields, "language")
|
||||
assert.Empty(t, got.language)
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
url, got, _ := upstream(t)
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,
|
||||
ModelName: tc.model, APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = a.Transcribe(WithLanguage(context.Background(), "zh"), []byte("RIFF"), "a.wav")
|
||||
require.NoError(t, err)
|
||||
tc.check(t, got)
|
||||
})
|
||||
}
|
||||
|
||||
// "auto" is every vendor's default and is not sent.
|
||||
url, got, _ := upstream(t)
|
||||
a, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1", ModelName: "whisper-1", APIKey: "k",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = a.Transcribe(WithLanguage(context.Background(), "auto"), []byte("RIFF"), "a.wav")
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, got.fields, "language")
|
||||
}
|
||||
|
||||
func TestEmptyAudioIsAnError(t *testing.T) {
|
||||
url, _, calls := upstream(t)
|
||||
a, err := NewASR(&Config{Source: types.ModelSourceRemote, Provider: "generic", BaseURL: url, ModelName: "m"})
|
||||
require.NoError(t, err)
|
||||
_, err = a.Transcribe(context.Background(), nil, "a.wav")
|
||||
assert.Error(t, err)
|
||||
assert.Zero(t, calls.Load())
|
||||
}
|
||||
|
||||
func TestInternalBaseURLIsRefused(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "")
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
_, err := NewASR(&Config{
|
||||
Source: types.ModelSourceRemote, Provider: "generic",
|
||||
BaseURL: "http://169.254.169.254/latest/meta-data/", ModelName: "asr-test",
|
||||
})
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
|
@ -346,6 +346,8 @@ type VendorCompat struct {
|
|||
Rerank RerankCompat `json:"rerank,omitempty"`
|
||||
// Embeddings is the embedding protocol overlay, likewise one per vendor.
|
||||
Embeddings EmbeddingsCompat `json:"embeddings,omitempty"`
|
||||
// Transcriptions is the speech-to-text overlay, likewise one per vendor.
|
||||
Transcriptions TranscriptionsCompat `json:"transcriptions,omitempty"`
|
||||
}
|
||||
|
||||
// apply writes the set fields of an overlay struct onto a settings struct
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ func Register(v *Vendor) {
|
|||
if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) {
|
||||
v.RerankAPI = api.RerankCohere
|
||||
}
|
||||
if v.TranscriptionAPI == "" && v.SupportsType(types.ModelTypeASR) {
|
||||
v.TranscriptionAPI = api.TranscriptionOpenAI
|
||||
}
|
||||
if v.EmbeddingAPI == "" && v.SupportsType(types.ModelTypeEmbedding) {
|
||||
v.EmbeddingAPI = api.EmbeddingOpenAI
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,6 +81,11 @@ type Resolved struct {
|
|||
// for the same reason.
|
||||
EmbeddingAPI api.EmbeddingAPI
|
||||
Embeddings EmbeddingsSettings
|
||||
|
||||
// TranscriptionAPI and Transcriptions are filled only for an ASR
|
||||
// reference.
|
||||
TranscriptionAPI api.TranscriptionAPI
|
||||
Transcriptions TranscriptionsSettings
|
||||
}
|
||||
|
||||
// Resolve merges the vendor, catalog entry, extra-config and per-row
|
||||
|
|
@ -122,6 +127,9 @@ func Resolve(ref Ref) (*Resolved, error) {
|
|||
if modelType == types.ModelTypeEmbedding {
|
||||
return resolveEmbeddings(ref, vendor, spec, cataloged, baseURL)
|
||||
}
|
||||
if modelType == types.ModelTypeASR {
|
||||
return resolveTranscriptions(ref, vendor, spec, cataloged, baseURL)
|
||||
}
|
||||
|
||||
resolvedAPI := spec.API
|
||||
if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" {
|
||||
|
|
@ -537,3 +545,62 @@ func resolveEmbeddings(
|
|||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolveTranscriptions merges the speech-to-text layers, in the same order
|
||||
// as resolveEmbeddings: protocol default, vendor compat, the catalog entry,
|
||||
// the row's own compat, then remote_model_name.
|
||||
func resolveTranscriptions(
|
||||
ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string,
|
||||
) (*Resolved, error) {
|
||||
protocol := vendor.TranscriptionAPI
|
||||
if protocol == "" {
|
||||
protocol = api.TranscriptionOpenAI
|
||||
}
|
||||
settings := DefaultTranscriptions()
|
||||
apply(&settings, &vendor.Compat.Transcriptions)
|
||||
for _, raw := range []json.RawMessage{spec.Compat, ref.Override.CompatJSON()} {
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
overlay := &TranscriptionsCompat{}
|
||||
if err := decodeCompat(raw, overlay); err != nil {
|
||||
return nil, fmt.Errorf("transcriptions compat: %w", err)
|
||||
}
|
||||
apply(&settings, overlay)
|
||||
}
|
||||
if settings.API != "" {
|
||||
protocol = settings.API
|
||||
}
|
||||
if !protocol.Known() {
|
||||
return nil, fmt.Errorf("catalog: unknown transcription api %q for %s/%s", protocol, vendor.ID, spec.ID)
|
||||
}
|
||||
settings.API = protocol
|
||||
switch settings.LanguageParam {
|
||||
case "", LanguageForm, LanguageHeader, LanguageASROptions:
|
||||
default:
|
||||
return nil, fmt.Errorf("catalog: unknown language_param %q for %s/%s",
|
||||
settings.LanguageParam, vendor.ID, spec.ID)
|
||||
}
|
||||
// Checked after every layer: an entry may lift a vendor-wide refusal for
|
||||
// the one model that takes the audio in the request.
|
||||
if settings.UnsupportedReason != "" {
|
||||
return nil, fmt.Errorf(
|
||||
"catalog: %s does not serve %q through a protocol this build implements: %s",
|
||||
vendor.ID, spec.ID, settings.UnsupportedReason,
|
||||
)
|
||||
}
|
||||
|
||||
out := &Resolved{
|
||||
Vendor: vendor,
|
||||
Spec: spec,
|
||||
Cataloged: cataloged,
|
||||
BaseURL: baseURL,
|
||||
RemoteModel: ref.Model,
|
||||
TranscriptionAPI: protocol,
|
||||
Transcriptions: settings,
|
||||
}
|
||||
if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" {
|
||||
out.RemoteModel = override
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
package catalog
|
||||
|
||||
import "github.com/Tencent/WeKnora/internal/models/api"
|
||||
|
||||
// Where a transcription protocol puts the operator's language hint. Each
|
||||
// vendor documents one place, or none.
|
||||
const (
|
||||
// LanguageForm is a multipart field named language (OpenAI's shape,
|
||||
// ISO-639-1).
|
||||
LanguageForm = "form"
|
||||
// LanguageHeader is an HTTP request header named language (MiniMax,
|
||||
// BCP-47).
|
||||
LanguageHeader = "header"
|
||||
// LanguageASROptions is asr_options.language in a chat-served request
|
||||
// body (Alibaba, Xiaomi).
|
||||
LanguageASROptions = "asr_options"
|
||||
)
|
||||
|
||||
// TranscriptionsCompat is the overlay form (every field optional) of the
|
||||
// speech-to-text settings. JSON keys are the names used in models.json.
|
||||
type TranscriptionsCompat struct {
|
||||
API *api.TranscriptionAPI `json:"api,omitempty"`
|
||||
Path *string `json:"path,omitempty"`
|
||||
// ResponseFormat is sent as response_format when non-empty. OpenAI's
|
||||
// gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json, which is
|
||||
// also the default, so most models leave it unset.
|
||||
ResponseFormat *string `json:"response_format,omitempty"`
|
||||
// LanguageParam names where the language hint goes: form, header or
|
||||
// asr_options; empty sends none.
|
||||
LanguageParam *string `json:"language_param,omitempty"`
|
||||
// MaxFileBytes is the documented upload ceiling for the audio itself.
|
||||
MaxFileBytes *int `json:"max_file_bytes,omitempty"`
|
||||
// MaxEncodedBytes is the documented ceiling on the base64 data URI, for
|
||||
// the protocols that send the audio that way.
|
||||
MaxEncodedBytes *int `json:"max_encoded_bytes,omitempty"`
|
||||
// Formats lists the file extensions the vendor documents, without the
|
||||
// dot. Empty means the vendor documents no closed list.
|
||||
Formats []string `json:"formats,omitempty"`
|
||||
RequestTimeout *int `json:"request_timeout_seconds,omitempty"`
|
||||
// UnsupportedReason marks a model this build cannot call.
|
||||
UnsupportedReason *string `json:"unsupported_reason,omitempty"`
|
||||
}
|
||||
|
||||
// TranscriptionsSettings is the resolved (fully defaulted) form.
|
||||
type TranscriptionsSettings struct {
|
||||
API api.TranscriptionAPI
|
||||
Path string
|
||||
ResponseFormat string
|
||||
LanguageParam string
|
||||
// MaxFileBytes and MaxEncodedBytes refuse an upload the vendor would
|
||||
// reject anyway, before sending it; 0 leaves the check to the vendor.
|
||||
MaxFileBytes int
|
||||
MaxEncodedBytes int
|
||||
Formats []string
|
||||
// RequestTimeout caps one request, in seconds.
|
||||
RequestTimeout int
|
||||
// UnsupportedReason is set where a vendor's recognition models, or some
|
||||
// of them, cannot be called with the audio in the request — they take a
|
||||
// public URL or run as asynchronous tasks. Resolve refuses such a model
|
||||
// rather than sending it a shape it does not accept.
|
||||
UnsupportedReason string
|
||||
}
|
||||
|
||||
// DefaultTranscriptions is the protocol baseline: a form with file and model
|
||||
// and nothing else, which every speech-to-text endpoint in this catalog
|
||||
// documents. The deadline is the one the pre-catalog client carried; audio
|
||||
// transcription is slow.
|
||||
func DefaultTranscriptions() TranscriptionsSettings {
|
||||
return TranscriptionsSettings{Path: "/audio/transcriptions", RequestTimeout: 300}
|
||||
}
|
||||
|
|
@ -251,6 +251,9 @@ type Vendor struct {
|
|||
// OpenAI shape for any vendor that serves embeddings without naming
|
||||
// another.
|
||||
EmbeddingAPI api.EmbeddingAPI
|
||||
// TranscriptionAPI is the speech-to-text protocol. Register defaults it
|
||||
// to the OpenAI shape for any vendor that serves ASR.
|
||||
TranscriptionAPI api.TranscriptionAPI
|
||||
// DefaultBaseURLs by model type; GetDefaultURL falls back to chat.
|
||||
DefaultBaseURLs map[types.ModelType]string
|
||||
ModelTypes []types.ModelType
|
||||
|
|
|
|||
|
|
@ -14,10 +14,9 @@ import (
|
|||
// protocol, a bad compat key or an invalid thinking level is rejected where
|
||||
// it is written instead of surfacing as a failed chat call much later.
|
||||
//
|
||||
// Chat, VLM, embedding and rerank rows are checked; all four are built from
|
||||
// catalog.Resolve, so a row that does not resolve cannot be used. ASR rows
|
||||
// return nil: the ASR client does not resolve through the catalog yet, so a
|
||||
// spec stored on such a row is inert — neither validated nor applied.
|
||||
// Every model type is checked: all of them are built from catalog.Resolve,
|
||||
// so a row that does not resolve cannot be used and should fail the save
|
||||
// rather than the first call.
|
||||
func ValidateRow(modelName string, modelType types.ModelType, params *types.ModelParameters) error {
|
||||
if params == nil || !resolvesThroughCatalog(modelType) {
|
||||
return nil
|
||||
|
|
@ -44,7 +43,8 @@ func ValidateRow(modelName string, modelType types.ModelType, params *types.Mode
|
|||
|
||||
func resolvesThroughCatalog(modelType types.ModelType) bool {
|
||||
switch modelType {
|
||||
case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank:
|
||||
case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank,
|
||||
types.ModelTypeASR:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
|
|
|||
|
|
@ -56,9 +56,17 @@ func TestValidateRow(t *testing.T) {
|
|||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "asr rows are not resolved through the catalog", modelName: "whisper-1",
|
||||
name: "a catalogued asr row resolves", modelName: "whisper-1",
|
||||
modelType: types.ModelTypeASR,
|
||||
params: &types.ModelParameters{Provider: "openai", Spec: &types.ModelSpecOverride{API: "nope"}},
|
||||
params: &types.ModelParameters{Provider: "openai"},
|
||||
},
|
||||
{
|
||||
name: "unknown asr compat key is rejected", modelName: "whisper-1",
|
||||
modelType: types.ModelTypeASR,
|
||||
params: &types.ModelParameters{
|
||||
Provider: "openai", Spec: &types.ModelSpecOverride{Compat: map[string]any{"dimensions_field": "x"}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "a catalogued chat row resolves", modelName: "deepseek-v4-pro",
|
||||
|
|
|
|||
|
|
@ -55,6 +55,8 @@
|
|||
{"name": "qwen2.5-vl-embedding (fused vectors only)", "match": "qwen2.5-vl-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "dimension", "max_batch_size": 1}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},
|
||||
{"name": "multimodal-embedding-v1", "match": "multimodal-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "", "max_batch_size": 20}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},
|
||||
{"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
|
||||
{"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}
|
||||
{"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
|
||||
{"id": "qwen3-asr-flash", "name": "Qwen3 ASR Flash", "type": "ASR", "compat": {"api": "openai-chat-audio", "max_encoded_bytes": 10485760, "language_param": "asr_options"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"},
|
||||
{"name": "Other Alibaba recognition models (not callable with the audio in the request)", "match": "*", "type": "ASR", "compat": {"unsupported_reason": "Alibaba's other recognition models take a public file URL or run as asynchronous tasks; only qwen3-asr-flash accepts the audio in the request"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -136,6 +136,7 @@ func init() {
|
|||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
RerankAPI: api.RerankDashScope,
|
||||
// Text and multimodal embeddings live under different roots of the
|
||||
|
|
@ -160,6 +161,13 @@ func init() {
|
|||
return root + "/compatible-mode/v1/embeddings", nil
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
// ASR: qwen3-asr-flash is the one recognition model callable with
|
||||
// the audio in the request, on the compatible chat endpoint as a
|
||||
// base64 data URI; its catalog entry declares that. Every other
|
||||
// ASR name falls to a catch-all entry that refuses it: Paraformer,
|
||||
// Fun-ASR and the *-filetrans models are asynchronous tasks that
|
||||
// take a public file URL, which no protocol here can send
|
||||
// (https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference).
|
||||
// Text models: the OpenAI-compatible endpoint
|
||||
// (https://help.aliyun.com/zh/model-studio/embedding-interfaces-compatible-with-openai),
|
||||
// which takes model, input, dimensions and encoding_format.
|
||||
|
|
|
|||
|
|
@ -124,14 +124,16 @@ func init() {
|
|||
types.ModelTypeEmbedding: BaseURL,
|
||||
types.ModelTypeVLLM: BaseURL,
|
||||
},
|
||||
// ASR is deliberately absent: Azure serves Whisper, but
|
||||
// internal/models/asr builds every remote client with a plain
|
||||
// OpenAI config (Bearer auth, {base}/audio/transcriptions) and has
|
||||
// no provider or extra-config field to route through this vendor's
|
||||
// api-key header and deployments path. Advertising the type would
|
||||
// let the picker create a row that fails at first use. The Endpoint
|
||||
// hook below already knows the ASR path, so re-adding the type is a
|
||||
// one-line change once asr.Config carries the provider.
|
||||
// ASR is deliberately absent. The ASR client now resolves through
|
||||
// this vendor (api-key header, Endpoint hook below), but the only
|
||||
// v1 reference for audio is the preview one —
|
||||
// POST {endpoint}/openai/v1/audio/transcriptions?api-version=preview
|
||||
// (https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest)
|
||||
// — while the hook sends a row without api_version to the v1 path
|
||||
// with no version at all. Until that is verified against a live
|
||||
// resource, advertising the type would let the picker create a row
|
||||
// that may fail at first use. A row with an explicit api_version
|
||||
// would take the documented deployments path.
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
|
|
|
|||
|
|
@ -108,6 +108,17 @@ func init() {
|
|||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}},
|
||||
Compat: catalog.VendorCompat{
|
||||
// Transcriptions keeps the baseline, json by default. The
|
||||
// pre-catalog client always asked for verbose_json, which not every
|
||||
// model or server can produce: OpenAI's gpt-4o transcribers accept
|
||||
// only json, and vox-box's FunASR backend answers verbose_json with
|
||||
// a bare JSON string. A row that wants segments sets
|
||||
// {"response_format": "verbose_json"} in its compat. The language
|
||||
// hint goes as OpenAI's language form field, which a server that
|
||||
// does not implement it ignores.
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageForm),
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// Whatever the operator runs. Everything the pre-catalog client
|
||||
// sent stays: vLLM, SGLang, TEI and Ollama's OpenAI route all
|
||||
|
|
|
|||
|
|
@ -118,6 +118,17 @@ func init() {
|
|||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}},
|
||||
Compat: catalog.VendorCompat{
|
||||
// Transcriptions keeps the baseline, json by default. GPUStack
|
||||
// serves audio through vox-box (https://github.com/gpustack/vox-box),
|
||||
// whose route wraps a json answer as {"text": ...} but returns
|
||||
// verbose_json as whatever the backend produced — and its FunASR
|
||||
// backend (SenseVoice, Paraformer) produces a bare string for every
|
||||
// format. The pre-catalog client's verbose_json therefore arrived
|
||||
// as a JSON string that could not be decoded. The route reads a
|
||||
// language form field.
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageForm),
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// vLLM's embedding server: dimensions for Matryoshka models, the
|
||||
// row's truncation budget, and encoding_format.
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ func init() {
|
|||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
ExtraFields: []catalog.ExtraField{{
|
||||
Key: catalog.ExtraScoreScale,
|
||||
|
|
@ -104,6 +105,9 @@ func init() {
|
|||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}},
|
||||
Compat: catalog.VendorCompat{
|
||||
// Transcriptions keeps the baseline: the proxy serves the OpenAI
|
||||
// /audio/transcriptions route for whichever upstream it is
|
||||
// configured with (https://docs.litellm.ai/docs/audio_transcription).
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// https://docs.litellm.ai/docs/embedding/supported_embedding:
|
||||
// model, input, user, dimensions, encoding_format; anything else
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
{"id": "mimo-v2.5-pro", "name": "MiMo V2.5 Pro", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.435, "output": 0.87, "cache_read": 0.0036}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},
|
||||
{"id": "mimo-v2.5", "name": "MiMo V2.5", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.14, "output": 0.28, "cache_read": 0.0028}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},
|
||||
{"id": "mimo-v2.5-pro-ultraspeed", "name": "MiMo V2.5 Pro Ultraspeed", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 1.305, "output": 2.61, "cache_read": 0.0108}, "source": "https://mimo.mi.com/static/docs/news/latest/beta-extended.md"},
|
||||
{"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"}
|
||||
{"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"},
|
||||
{"id": "mimo-v2.5-asr", "name": "MiMo V2.5 ASR", "type": "ASR", "source": "https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,10 +71,22 @@ func init() {
|
|||
DefaultBaseURLs: map[types.ModelType]string{
|
||||
types.ModelTypeKnowledgeQA: BaseURL,
|
||||
},
|
||||
TranscriptionAPI: api.TranscriptionChatAudio,
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
// ASR: mimo-v2.5-asr on the chat endpoint, WAV or MP3 as a base64
|
||||
// data URI whose "encoded string size must not exceed 10 MB"
|
||||
// (https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition).
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// The ceiling counts the whole data URI as sent.
|
||||
MaxEncodedBytes: catalog.Ptr(10 << 20),
|
||||
Formats: []string{"wav", "mp3"},
|
||||
// asr_options.language: "auto|zh|en".
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageASROptions),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
|
||||
PromptCacheAccounting: catalog.Ptr(true),
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
{"id": "MiniMax-M2.5-highspeed", "name": "MiniMax M2.5 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.6, "output": 2.4, "cache_read": 0.06, "cache_write": 0.375}, "thinking_levels": {"off": null}},
|
||||
{"id": "MiniMax-M2.1", "name": "MiniMax M2.1", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.03, "cache_write": 0.375}, "thinking_levels": {"off": null}},
|
||||
{"id": "MiniMax-M2.1-highspeed", "name": "MiniMax M2.1 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "thinking_levels": {"off": null}, "source": "https://platform.minimax.io/docs/guides/text-generation"},
|
||||
{"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}}
|
||||
{"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}},
|
||||
{"id": "asr-1.0", "name": "MiniMax ASR 1.0", "type": "ASR", "source": "https://platform.minimax.io/docs/api-reference/speech-to-text"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,8 +95,23 @@ func init() {
|
|||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// https://platform.minimax.io/docs/api-reference/speech-to-text
|
||||
// (and the same page on platform.minimax.cn): multipart model
|
||||
// (asr-1.0) + file on /v1/speech_to_text rather than the OpenAI
|
||||
// path, answering {text, duration, trace_id} for the default
|
||||
// json. At most 50 MB and 500 seconds; "超出会返回 400 而不会被截断".
|
||||
Path: catalog.Ptr("/speech_to_text"),
|
||||
MaxFileBytes: catalog.Ptr(50 << 20),
|
||||
// "wav / aiff / flac / alac(m4a) / mp3 / aac / opus / ogg";
|
||||
// "不支持无容器的裸 PCM 数据". The language hint is a request
|
||||
// header, a BCP-47 tag, not a form field.
|
||||
Formats: []string{"wav", "aiff", "flac", "m4a", "mp3", "aac", "opus", "ogg"},
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageHeader),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
|
||||
ThinkingEnabledValue: catalog.Ptr("adaptive"),
|
||||
|
|
|
|||
|
|
@ -29,6 +29,10 @@
|
|||
|
||||
{"id": "text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8191, "cost": {"input": 0.02, "output": 0}},
|
||||
{"id": "text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8191, "cost": {"input": 0.13, "output": 0}},
|
||||
{"id": "whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
|
||||
{"id": "gpt-transcribe", "name": "GPT Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
|
||||
{"id": "gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
|
||||
{"id": "gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
|
||||
{"id": "text-embedding-ada-002", "name": "text-embedding-ada-002", "type": "Embedding", "dimension": 1536, "context_window": 8192, "cost": {"input": 0.1, "output": 0}, "compat": {"dimensions_field": ""}}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -97,6 +97,19 @@ func init() {
|
|||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// The transcription reference (URL in openaitranscriptions):
|
||||
// response_format differs per model — "For gpt-4o-transcribe
|
||||
// and gpt-4o-mini-transcribe, the only supported format is
|
||||
// json", the default — so it is declared per entry, not here.
|
||||
// "Files can be up to 25 MB"
|
||||
// (https://developers.openai.com/api/docs/guides/speech-to-text).
|
||||
MaxFileBytes: catalog.Ptr(25 << 20),
|
||||
// "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm", and a
|
||||
// language field in ISO-639-1.
|
||||
Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageForm),
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// https://developers.openai.com/api/reference/resources/embeddings/methods/create:
|
||||
// model, input, dimensions (text-embedding-3 and later; the ada-002
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
{"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}},
|
||||
{"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
|
||||
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
|
||||
{"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}
|
||||
{"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"},
|
||||
{"id": "openai/whisper-large-v3", "name": "Whisper Large v3", "type": "ASR", "source": "https://openrouter.ai/docs/api/api-reference/stt/create-transcription"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,8 +75,18 @@ func init() {
|
|||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// https://openrouter.ai/docs/api/api-reference/stt/create-transcription:
|
||||
// OpenAI-style multipart file + model, json by default. "Max 25
|
||||
// MB; send larger files as base64 JSON via input_audio."
|
||||
MaxFileBytes: catalog.Ptr(25 << 20),
|
||||
// language is an ISO-639-1 form field. The format "is derived
|
||||
// from the filename extension", with no closed list.
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageForm),
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// https://openrouter.ai/docs/api/api-reference/embeddings/create-embeddings:
|
||||
// model, input, dimensions, encoding_format, input_type, provider,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@
|
|||
{"id": "google/gemini-3.6-flash", "name": "Gemini 3.6 Flash", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 65536, "thinking_levels": {"off": null, "minimal": "min", "xhigh": "high", "max": "high"}},
|
||||
{"id": "deepseek/deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro", "reasoning": true, "input": ["text"], "context_window": 1000000, "max_output_tokens": 131072},
|
||||
{"id": "zai/glm-5.2", "name": "GLM-5.2", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072},
|
||||
{"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144}
|
||||
{"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144},
|
||||
{"id": "openai/gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},
|
||||
{"id": "openai/gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},
|
||||
{"id": "openai/whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,8 +77,19 @@ func init() {
|
|||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create:
|
||||
// OpenAI-compatible multipart, json by default, verbose_json
|
||||
// available. "The maximum upload size per request is 32 MB."
|
||||
MaxFileBytes: catalog.Ptr(32 << 20),
|
||||
// "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, webm"; language
|
||||
// is an ISO 639-1 form field.
|
||||
Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},
|
||||
LanguageParam: catalog.Ptr(catalog.LanguageForm),
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// https://docs.requesty.ai/api-reference/endpoint/embeddings-create:
|
||||
// model, input, encoding_format, dimensions.
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
{"id": "Qwen/Qwen3-Embedding-0.6B", "name": "Qwen3 Embedding 0.6B", "type": "Embedding", "dimension": 1024, "source": "https://api-docs.siliconflow.cn/docs/api/embeddings-post"},
|
||||
{"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker v2 M3", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},
|
||||
{"id": "Qwen/Qwen3-Reranker-8B", "name": "Qwen3 Reranker 8B", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},
|
||||
{"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://docs.siliconflow.cn/cn/api-reference/audio/create-audio-transcriptions"}
|
||||
{"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"},
|
||||
{"id": "TeleAI/TeleSpeechASR", "name": "TeleSpeechASR", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,6 +77,12 @@ func init() {
|
|||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post:
|
||||
// file and model only — no response_format — and a file of
|
||||
// "时长不超过 1 小时,文件大小不超过 50MB".
|
||||
MaxFileBytes: catalog.Ptr(50 << 20),
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// https://api-docs.siliconflow.cn/docs/api/embeddings-post: model,
|
||||
// input, encoding_format, and dimensions "仅 Qwen/Qwen3 系列支持"
|
||||
|
|
|
|||
|
|
@ -110,6 +110,9 @@ func TestEveryCatalogEntryResolves(t *testing.T) {
|
|||
if modelType == types.ModelTypeKnowledgeQA && !r.API.Known() {
|
||||
t.Errorf("%s/%s: resolved to unknown API %q", id, name, r.API)
|
||||
}
|
||||
if modelType == types.ModelTypeASR && !r.TranscriptionAPI.Known() {
|
||||
t.Errorf("%s/%s: resolved to unknown transcription API %q", id, name, r.TranscriptionAPI)
|
||||
}
|
||||
if modelType == types.ModelTypeEmbedding && !r.EmbeddingAPI.Known() {
|
||||
t.Errorf("%s/%s: resolved to unknown embedding API %q", id, name, r.EmbeddingAPI)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
{"id": "glm-4.5-air", "name": "GLM-4.5 Air", "reasoning": true, "input": ["text"], "context_window": 131072, "max_output_tokens": 98304, "cost": {"input": 0.2, "output": 1.1, "cache_read": 0.03}, "compat": {"supports_reasoning_effort": false}},
|
||||
{"id": "embedding-3", "name": "Embedding-3", "type": "Embedding", "dimension": 2048, "compat": {"max_batch_size": 64}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},
|
||||
{"id": "embedding-2", "name": "Embedding-2", "type": "Embedding", "dimension": 1024, "compat": {"dimensions_field": ""}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},
|
||||
{"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"}
|
||||
{"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"},
|
||||
{"id": "glm-asr-2512", "name": "GLM-ASR-2512 (at most 30 s per file)", "type": "ASR", "source": "https://docs.bigmodel.cn/api-reference/模型-api/语音转文本"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,8 +107,19 @@ func init() {
|
|||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Transcriptions: catalog.TranscriptionsCompat{
|
||||
// https://docs.bigmodel.cn/api-reference/模型-api/语音转文本:
|
||||
// POST {base}/audio/transcriptions, multipart file + model
|
||||
// (glm-asr-2512), answering {text}. The file is ".wav / .mp3",
|
||||
// at most 25 MB and 30 seconds — the duration cannot be checked
|
||||
// here without decoding, so a longer file is the vendor's error.
|
||||
MaxFileBytes: catalog.Ptr(25 << 20),
|
||||
// No language field; prompt and hotwords are its only hints.
|
||||
Formats: []string{"wav", "mp3"},
|
||||
},
|
||||
Embeddings: catalog.EmbeddingsCompat{
|
||||
// https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入: model,
|
||||
// input, dimensions — no encoding_format. embedding-2 is fixed at
|
||||
|
|
|
|||
|
|
@ -44,44 +44,3 @@ func ApplyCustomHeaders(req *http.Request, headers map[string]string) {
|
|||
req.Header.Set(name, v)
|
||||
}
|
||||
}
|
||||
|
||||
// CustomHeadersRoundTripper 是一个 http.RoundTripper 包装器,
|
||||
// 会在每个 HTTP 请求发出前注入用户自定义的 header。
|
||||
// 用于无法直接拿到底层 *http.Request 的场景(如 go-openai SDK)。
|
||||
type CustomHeadersRoundTripper struct {
|
||||
Headers map[string]string
|
||||
Base http.RoundTripper
|
||||
}
|
||||
|
||||
// RoundTrip 实现 http.RoundTripper 接口。
|
||||
func (t *CustomHeadersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
base := t.Base
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
if len(t.Headers) > 0 {
|
||||
// 复制请求以避免修改调用方传入的对象。
|
||||
cloned := req.Clone(req.Context())
|
||||
ApplyCustomHeaders(cloned, t.Headers)
|
||||
return base.RoundTrip(cloned)
|
||||
}
|
||||
return base.RoundTrip(req)
|
||||
}
|
||||
|
||||
// WrapHTTPClientWithHeaders 返回一个新的 *http.Client,在原有 client 基础上注入自定义 header。
|
||||
// 如果 headers 为空则直接返回原 client,避免不必要的开销。
|
||||
func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) *http.Client {
|
||||
if len(headers) == 0 {
|
||||
return client
|
||||
}
|
||||
if client == nil {
|
||||
client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig())
|
||||
}
|
||||
base := client.Transport
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
wrapped := *client
|
||||
wrapped.Transport = &CustomHeadersRoundTripper{Headers: headers, Base: base}
|
||||
return &wrapped
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -48,47 +45,3 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) {
|
|||
t.Fatalf("unexpected headers added: %+v", req.Header)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapHTTPClientWithHeaders(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")
|
||||
ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(ResetSSRFWhitelistForTest)
|
||||
gotTrace := ""
|
||||
gotAuth := ""
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotTrace = r.Header.Get("X-Trace-Id")
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
io.Copy(io.Discard, r.Body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := WrapHTTPClientWithHeaders(nil, map[string]string{
|
||||
"X-Trace-Id": "rt-1",
|
||||
"Authorization": "Bearer should-not-override",
|
||||
})
|
||||
|
||||
req, _ := http.NewRequest("POST", srv.URL, strings.NewReader("{}"))
|
||||
req.Header.Set("Authorization", "Bearer kept")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if gotTrace != "rt-1" {
|
||||
t.Fatalf("expected custom header injected, got %q", gotTrace)
|
||||
}
|
||||
if gotAuth != "Bearer kept" {
|
||||
t.Fatalf("reserved header must not be overridden, got %q", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapHTTPClientWithHeaders_EmptyReturnsOriginal(t *testing.T) {
|
||||
orig := &http.Client{}
|
||||
wrapped := WrapHTTPClientWithHeaders(orig, nil)
|
||||
if wrapped != orig {
|
||||
t.Fatalf("expected original client returned when headers empty")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ WeKnora 采用"主服务 + 前端 + 文档解析微服务"的三进程核心架
|
|||
| 配置 | `github.com/spf13/viper` + `config/config.yaml` + 环境变量 | — |
|
||||
| 可观测 | OpenTelemetry + Langfuse(`internal/tracing/langfuse`) | LLM 调用级 trace |
|
||||
| gRPC | `google.golang.org/grpc` v1.81.0 | 调用 docreader |
|
||||
| LLM 接入 | `sashabaranov/go-openai`、Ollama、腾讯云 LKE 等 | 18+ 模型提供商(OpenAI 兼容 / Ollama / 云厂商 SDK) |
|
||||
| LLM 接入 | 自研协议层 `internal/models/api`(OpenAI / Anthropic / Gemini / DashScope 等线格式各一个包)、Ollama、腾讯云 LKE SDK 等 | 27 家厂商由 `internal/models/vendors` 声明,见[模型管理](../03-features/06-models.md) |
|
||||
| 向量/检索 | pgvector、ES v7/v8、OpenSearch、Qdrant、Milvus、Weaviate、Doris、腾讯 VectorDB、sqlite-vec | 由 `RETRIEVE_DRIVER` 与 `vector_stores` 表动态装配 |
|
||||
| 知识图谱 | `neo4j-go-driver/v6` | 可选 |
|
||||
| 数据分析 | DuckDB(`duckdb-go/v2`)、`pg_query_go` SQL 校验 | Agent 数据分析工具 |
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ const (
|
|||
| `Embedding` | `embedding` | `internal/models/embedding` | `Embed` / `BatchEmbed`(含 `GetDimensions`) | 文本向量化,供向量检索索引与查询 |
|
||||
| `Rerank` | `rerank` | `internal/models/rerank` | `Rerank(query, documents)` 返回 `RankResult` | 检索结果精排 |
|
||||
| `VLLM` | `vllm` | `internal/models/vlm` | `Predict(imgBytes, prompt)` | 视觉语言模型(VLM),文档图片理解 / 多模态解析 |
|
||||
| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本与分段时间戳 | 音频转写(自动语音识别) |
|
||||
| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本;模型提供时附带分段时间戳(如 OpenAI `whisper-1`) | 音频转写(自动语音识别) |
|
||||
|
||||
前后端类型映射见 `internal/handler/model.go` 的 `modelTypeToFrontend()`(`KnowledgeQA -> chat` 等)。
|
||||
|
||||
|
|
@ -176,7 +176,7 @@ builtin_models:
|
|||
|
||||
| 层 | 位置 | 职责 |
|
||||
|----|------|------|
|
||||
| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`。各自持有请求/响应结构与解析,不含任何厂商名 |
|
||||
| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`;语音:`openaitranscriptions`。各自持有请求/响应结构与解析,不含任何厂商名 |
|
||||
| 厂商层 | `internal/models/vendors/<id>/` | 一个厂商一个目录:`vendor.go`(注册 `catalog.Vendor`)、`models.json`(模型目录)、`icon.svg`(品牌图标,`go:embed` 进二进制) |
|
||||
| 目录层 | `internal/models/catalog` | 合并厂商定义、模型条目、部署叠加与单行覆盖,`Resolve` 得出「这个模型到底怎么发请求」 |
|
||||
|
||||
|
|
@ -235,7 +235,7 @@ make model-catalog-check
|
|||
|
||||
前端不需要任何改动:厂商下拉、图标、额外字段、内置模型列表都由 `GET /api/v1/models/providers` 动态渲染。
|
||||
|
||||
写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(chat、VLM、embedding、rerank 四类;ASR 还没进目录),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。
|
||||
写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(全部模型类型),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。
|
||||
|
||||
#### 厂商更新了模型怎么办
|
||||
|
||||
|
|
@ -280,7 +280,21 @@ Embedding 行也有几处按厂商文档纠正的行为变化(逐厂商的出
|
|||
7. **OpenAI 兼容回复里没有 `index` 时按顺序取**(和旧实现一致);有 `index` 就按它放回,重复或缺位报错。
|
||||
8. **Jina 的 `task`、Gemini 的 `taskType`、OpenRouter 的 `input_type`、火山的 `instructions`、百炼原生接口的 `text_type` / `instruct` 都不发**。它们会改变文档侧向量,开启后同一个知识库里新旧向量不在同一空间;需要按行显式开启的设计另见 Tencent/WeKnora#1401。
|
||||
|
||||
另外 Azure OpenAI 不再声明支持 ASR(ASR 客户端只会构造标准 OpenAI 客户端,根本无法带上 Azure 的 `api-key` 头和部署路径,这类行此前就调不通)。
|
||||
ASR 行的变化(逐厂商的出站表单由 `internal/models/asr/wire_test.go` 钉住):
|
||||
|
||||
1. **不再一律发 `response_format=verbose_json`**。OpenAI 文档写明 gpt-4o-transcribe / gpt-4o-mini-transcribe「only supported format is json」,旧实现发给它们必然 400;GPUStack 的音频后端 vox-box 在 FunASR 模型(SenseVoice、Paraformer)上对 `verbose_json` 返回裸字符串,旧实现解不开。现在只有文档写明支持的模型声明它(OpenAI `whisper-1`),其余走默认 json,Langfuse 里的音频时长因此只在返回分段的模型上有。需要分段的自建行可以在 `spec.compat` 里写 `{"response_format": "verbose_json"}`。
|
||||
2. **按厂商文档的上限和格式在上传前拒绝**:文件大小(OpenAI / 智谱 / OpenRouter 25 MB、Requesty 32 MB、SiliconFlow / MiniMax 50 MB;阿里与小米按整段 `data:` URI 计 10 MB),以及文档给出封闭格式清单的厂商(智谱、小米只收 wav/mp3,OpenAI、Requesty、MiniMax 各有列表)。
|
||||
3. **回复里没有 `text` 字段即报错**,不再当成「未检测到语音」入库;静音音频返回的是空字符串 `text`,照常处理。
|
||||
4. ASR 行现在带上 provider 走目录解析,go-openai 依赖随之移除。provider 没有声明 ASR 的行:显式填了该厂商的直接报错;没填、靠 URL 识别到的,按 OpenAI 形状打到它的 URL(与旧实现一致),不走该厂商的 Endpoint 钩子。
|
||||
5. **支持 ASR 的厂商按文档补齐**。OpenAI 形状(multipart `file` + `model`):openai、siliconflow、gpustack、generic、智谱(`glm-asr-2512`,单文件 ≤30 秒)、MiniMax(`asr-1.0`,路径是 `/v1/speech_to_text`)、OpenRouter、Requesty、LiteLLM。经 chat completions 的 `input_audio`(base64 data URI,编码后 ≤10 MB):阿里云 `qwen3-asr-flash`、小米 `mimo-v2.5-asr`。
|
||||
6. **知识库的「音频语言提示」终于发出去了**(此前从未接线)。位置按厂商文档:OpenAI / Requesty / OpenRouter / GPUStack / generic 是 `language` 表单字段,MiniMax 是 `language` 请求头,阿里与小米是 `asr_options.language`;智谱、SiliconFlow、LiteLLM 文档没有这个参数,不发。填 `auto` 等同留空。
|
||||
7. **音频时长从回包里读**:MiniMax 的 `duration`、OpenAI 系与阿里的 `usage.seconds`,Langfuse 不再只靠 `verbose_json` 的分段。
|
||||
|
||||
阿里只有 `qwen3-asr-flash` 能直接带音频调用;其余 ASR 模型名由一条兜底条目拒绝并说明原因。
|
||||
|
||||
查过但没有接的:火山豆包语音(独立域名与密钥,当前文档的请求体只收音频 URL)、千帆(`vop_asr` 挂在应用实例下要 `app_id`;短语音接口 ≤60 秒且只收 pcm/wav/amr/m4a)、七牛(只收音频 URL)、Novita(GLM-ASR 自有接口,≤30 秒)、腾讯云 ASR(与混元不是同一套鉴权)、NVIDIA 托管 Riva(gRPC)、Gemini(没有专用转写接口,只能让对话模型听音频)、Azure(见下)。阿里 Paraformer / Fun-ASR / `*-filetrans` 是异步任务,同样要公网 URL。
|
||||
|
||||
Azure OpenAI 仍不声明 ASR:音频转写只出现在 v1 **preview** 参考里(`/openai/v1/audio/transcriptions?api-version=preview`),v1 GA 参考没有;未填 `api_version` 的行会被钩子发到不带版本的 v1 路径,能否调通没有核实。
|
||||
|
||||
### 模型调用链
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue