refactor(rerank): move the wire formats into protocol packages, and correct the vendor facts (#3488)

把 8 个手写 rerank client 收敛成 3 个协议包(internal/models/api/{cohererank,dashscoperank,nimrerank}),每个带一份从厂商文档逐字粘来的 golden fixture。LKEAP 和火山保留官方 SDK,但只实现 api.Reranker。rerank.NewReranker 改成 catalog.Resolve + 按协议分发,函数体里没有厂商名。生产代码净 -147 行。

三件原来每个 client 各管各、现在由共用层统一的事:分批(按厂商声明的上限,智谱 128 / NIM 512 等此前无人执行)、分数标度(NIM 与 GPUStack 返回 logit,其余为 0~1 概率,统一换算后再与 RerankThreshold 比较)、NIM 的 truncate(默认 NONE 会让超长 passage 整请求失败,此前从未发送)。

对照官方文档订正四处厂商事实:GPUStack 分数标度、千帆三项上限、阿里 500 条上限、qwen3-rerank 是第二种协议(隐藏并在构造期拒绝)。Novita / OpenRouter / LiteLLM 三家官方有 rerank 但未开类型,本次开放,零协议代码。

运行时行为变更:NVIDIA 起发 truncate: END;GPUStack 分数按 logit 换算(可按行覆盖);truncate_prompt_tokens 收窄为厂商声明制;qwen3-rerank 存量行从静默打错变为构造期报错。详见 PR 描述。

关联 #3486。
main
lyingbug 2026-09-21 10:36:26 +08:00 committed by GitHub
parent 9aa63c866f
commit d6c7665cb5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
57 changed files with 2875 additions and 1733 deletions

View File

@ -584,6 +584,7 @@ export function fabriTag(request: FabriTagRequest): Promise<FabriTagResponse> {
// Mirrors internal/models/catalog.ExtraField.
export interface ModelProviderExtraFieldOption {
label: string;
labels?: Record<string, string>;
value: string;
}
@ -613,6 +614,7 @@ export interface ModelProviderExtraField {
required?: boolean;
default?: string;
placeholder?: string;
placeholders?: Record<string, string>;
options?: ModelProviderExtraFieldOption[];
// Backend model types ("KnowledgeQA", "Embedding", "Rerank", "VLLM", "ASR"); empty = all.
model_types?: string[];

View File

@ -358,22 +358,24 @@
<div v-if="field.type === 'boolean'" class="vision-toggle">
<t-switch :model-value="extraConfigBool(field.key)"
@update:model-value="(v: boolean) => setExtraConfig(field.key, v ? 'true' : 'false')" />
<span v-if="field.placeholder" class="form-desc form-desc--inline">{{ field.placeholder }}</span>
<span v-if="extraFieldDisplayPlaceholder(field)" class="form-desc form-desc--inline">{{ extraFieldDisplayPlaceholder(field) }}</span>
</div>
<t-select v-else-if="field.type === 'select'" :model-value="formData.extraConfig[field.key] || ''"
:placeholder="field.placeholder || ''" clearable
:placeholder="extraFieldDisplayPlaceholder(field)" clearable
@update:model-value="(v: string) => setExtraConfig(field.key, v)">
<t-option v-for="opt in (field.options || [])" :key="opt.value" :value="opt.value" :label="opt.label || opt.value" />
<t-option v-for="opt in (field.options || [])" :key="opt.value" :value="opt.value"
:label="extraFieldDisplayOptionLabel(opt)" />
</t-select>
<t-input v-else-if="field.type === 'number'" :model-value="formData.extraConfig[field.key] || ''"
type="number" :placeholder="field.placeholder || ''"
type="number" :placeholder="extraFieldDisplayPlaceholder(field)"
@update:model-value="(v: string | number) => setExtraConfig(field.key, String(v ?? ''))" />
<t-input v-else-if="field.type === 'password'" :model-value="formData.extraConfig[field.key] || ''"
type="password" :placeholder="field.placeholder || ''" autocomplete="off" spellcheck="false"
type="password" :placeholder="extraFieldDisplayPlaceholder(field)" autocomplete="off" spellcheck="false"
@update:model-value="(v: string) => setExtraConfig(field.key, v)">
<template #prefix-icon><t-icon name="lock-on" /></template>
</t-input>
<t-input v-else :model-value="formData.extraConfig[field.key] || ''" :placeholder="field.placeholder || ''"
<t-input v-else :model-value="formData.extraConfig[field.key] || ''"
:placeholder="extraFieldDisplayPlaceholder(field)"
@update:model-value="(v: string) => setExtraConfig(field.key, v)" />
</div>
@ -599,7 +601,8 @@ import { MessagePlugin, DialogPlugin } from 'tdesign-vue-next'
import {
checkOllamaModels, checkRemoteModel, testEmbeddingModel, checkRerankModel, checkASRModel, listOllamaModels,
downloadOllamaModel, getDownloadProgress, checkOllamaStatus, resolveModelCatalog,
type OllamaModelInfo, type ModelProviderOption, type ModelProviderExtraField, type ModelCatalogEntry,
type OllamaModelInfo, type ModelProviderOption, type ModelProviderExtraField,
type ModelProviderExtraFieldOption, type ModelCatalogEntry,
type ResolvedModelCatalog,
} from '@/api/initialization'
import {
@ -615,6 +618,8 @@ import { useModelProvidersStore } from '@/stores/modelProviders'
import {
credentialLabelForModelType,
extraFieldLabel,
extraFieldOptionLabel,
extraFieldPlaceholder,
extraFieldsForModelType,
pickLocalized,
providerDescription,
@ -775,6 +780,10 @@ const matchTriggerWidth = (triggerElement: HTMLElement) => ({
})
const extraFieldDisplayLabel = (field: ModelProviderExtraField) => extraFieldLabel(field, currentLocale.value)
const extraFieldDisplayPlaceholder = (field: ModelProviderExtraField) =>
extraFieldPlaceholder(field, currentLocale.value)
const extraFieldDisplayOptionLabel = (option: ModelProviderExtraFieldOption) =>
extraFieldOptionLabel(option, currentLocale.value)
/**
* Vendors whose API is not a bearer-token API name their first credential

View File

@ -812,3 +812,66 @@ test('a vendor with no catalog falls back to a plain text field', async () => {
} finally { f2.close() }
} finally { f.close() }
})
// --- Vendor strings reach the form in the reader's language ---------------
// The editor renders whatever the catalog declares, so every operator-facing
// string a vendor ships has to go through the locale resolvers. Binding a raw
// `field.placeholder` or `opt.label` is silent: the form still works, it is
// just English in a Chinese product, and a test that only checks the catalog
// for a zh-CN entry passes while the editor ignores it. This reads the
// template instead.
test('every vendor-declared string in the form goes through a locale resolver', () => {
const template = descriptor.template?.content ?? ''
assert.ok(template.length > 0, 'the editor template should have been parsed')
// `field` is the extra-field loop variable, so any read of its placeholder
// is a vendor string rendered without resolving the locale. One input
// branch per field type plus the secret field means it is easy to bind four
// and miss the fifth, which is exactly what happened.
for (const pattern of [
/:placeholder="field\.placeholder/,
/\{\{\s*field\.placeholder\s*\}\}/,
/v-if="field\.placeholder"/,
]) {
assert.equal(
pattern.test(template), false,
`template reads field.placeholder directly (${pattern}); use extraFieldDisplayPlaceholder`,
)
}
// Options: scope to the extra-field select, because `opt` is also the loop
// variable of the model-type list and the catalog model list, whose labels
// come from i18n and from the catalog's own names.
const optionLoop = /v-for="opt in \(field\.options \|\| \[\]\)"[^>]*/.exec(template)
assert.ok(optionLoop, 'the extra-field select should loop over field.options')
assert.match(
optionLoop[0], /:label="extraFieldDisplayOptionLabel\(opt\)"/,
'extra-field option labels must resolve the locale',
)
// And the placeholder resolver is referenced, so the checks above cannot be
// satisfied by dropping the placeholders altogether.
assert.ok(template.includes('extraFieldDisplayPlaceholder('), 'placeholders must use the resolver')
})
test('extraFieldDisplayPlaceholder resolves the vendor placeholder for the active locale', async () => {
const f = await fixture({ type: 'rerank', providers: catalogProviders })
try {
const field = {
key: 'score_scale',
label: 'Rerank score scale',
type: 'select',
placeholder: 'match the reranker actually deployed behind this endpoint',
placeholders: { 'zh-CN': '按这个端点后面实际部署的重排模型选择' },
}
const rendered = f.vm.extraFieldDisplayPlaceholder(field)
assert.ok(
rendered === field.placeholder || rendered === field.placeholders['zh-CN'],
`expected one of the declared variants, got ${rendered}`,
)
assert.equal(f.vm.extraFieldDisplayPlaceholder({ key: 'k', label: 'k', type: 'string' }), '')
} finally {
f.close()
}
})

View File

@ -6,6 +6,8 @@ import type { ModelProviderOption } from '@/api/initialization'
import {
credentialLabelForModelType,
extraFieldLabel,
extraFieldOptionLabel,
extraFieldPlaceholder,
extraFieldsForModelType,
loadProvidersForType,
mergeProviderIndex,
@ -222,3 +224,37 @@ test('credentialLabelForModelType follows the vendor declaration, not a hardcode
assert.equal(pickLocalized(rerank?.labels, 'zh-CN', rerank!.label), 'SecretId(TC3 签名)')
assert.equal(pickLocalized(rerank?.labels, 'en-US', rerank!.label), 'SecretId')
})
// Select options and placeholders used to render their raw English string.
// That was invisible while every option was an identifier — LKEAP's region
// codes — and became a gap as soon as one was prose.
test('extraFieldOptionLabel resolves the locale, falling back to label then value', () => {
const option = {
label: '0..1 relevance (BGE class)',
labels: { 'zh-CN': '0~1 相关度(BGE 一类)' },
value: 'probability',
}
assert.equal(extraFieldOptionLabel(option, 'zh-CN'), '0~1 相关度(BGE 一类)')
assert.equal(extraFieldOptionLabel(option, 'en'), '0..1 relevance (BGE class)')
assert.equal(
extraFieldOptionLabel({ label: 'ap-guangzhou', value: 'ap-guangzhou' }, 'zh-CN'),
'ap-guangzhou',
)
assert.equal(extraFieldOptionLabel({ value: 'logit' }, 'zh-CN'), 'logit')
})
test('extraFieldPlaceholder resolves the locale and tolerates no placeholder', () => {
const field = {
key: 'score_scale',
label: 'Rerank score scale',
type: 'select',
placeholder: 'match the reranker actually deployed behind this endpoint',
placeholders: { 'zh-CN': '按这个端点后面实际部署的重排模型选择' },
}
assert.equal(extraFieldPlaceholder(field, 'zh-CN'), '按这个端点后面实际部署的重排模型选择')
assert.equal(
extraFieldPlaceholder(field, 'en'),
'match the reranker actually deployed behind this endpoint',
)
assert.equal(extraFieldPlaceholder({ key: 'k', label: 'k', type: 'string' }, 'zh-CN'), '')
})

View File

@ -109,6 +109,23 @@ export function extraFieldLabel(field: ModelProviderExtraField, locale?: string)
return pickLocalized(field.labels, locale, field.label || field.key)
}
/** Placeholder for the current locale, falling back to the default string. */
export function extraFieldPlaceholder(field: ModelProviderExtraField, locale?: string): string {
return pickLocalized(field.placeholders, locale, field.placeholder || '')
}
/**
* Option label for the current locale. Select options used to render their
* raw `label`, which was fine while every option was an identifier (a region
* code) and wrong as soon as one was prose.
*/
export function extraFieldOptionLabel(
option: { label?: string; labels?: Record<string, string>; value: string },
locale?: string,
): string {
return pickLocalized(option.labels, locale, option.label || option.value)
}
/** Stable display order: backend `order`, then label. */
export function sortProviders(providers: ReadonlyArray<ModelProviderOption>): ModelProviderOption[] {
return [...providers].sort((a, b) => {

View File

@ -0,0 +1,79 @@
package api
import (
"fmt"
"unicode/utf8"
)
// BatchLimits are the per-request ceilings a vendor documents. A zero field
// means the vendor states no limit, and nothing is enforced for it.
type BatchLimits struct {
// MaxItems caps how many documents one request may carry.
MaxItems int
// MaxItemRunes caps a single document. Vendors state these in characters
// or tokens; runes are the closest thing we can count without a
// tokenizer, and undercounting is the safe direction.
MaxItemRunes int
// MaxTotalRunes caps the whole request, query included.
MaxTotalRunes int
}
// Batch is one slice of the input, and where it started.
type Batch struct {
Start int
Items []string
}
// SplitBatches divides items into requests that satisfy limits. fixedRunes is
// charged to every batch (the query, which each request repeats).
//
// A single item that cannot fit on its own is an error rather than a silent
// truncation: the caller asked for this document to be scored, and scoring a
// prefix of it returns a number for something the user never sent.
func SplitBatches(items []string, fixedRunes int, limits BatchLimits) ([]Batch, error) {
if limits.MaxTotalRunes > 0 && fixedRunes >= limits.MaxTotalRunes {
return nil, fmt.Errorf(
"query is %d characters; the request budget is %d and must also fit a document",
fixedRunes, limits.MaxTotalRunes,
)
}
sizes := make([]int, len(items))
for i, item := range items {
sizes[i] = utf8.RuneCountInString(item)
if limits.MaxItemRunes > 0 && sizes[i] > limits.MaxItemRunes {
return nil, fmt.Errorf(
"document at index %d is %d characters; the per-document limit is %d",
i, sizes[i], limits.MaxItemRunes,
)
}
if limits.MaxTotalRunes > 0 && fixedRunes+sizes[i] > limits.MaxTotalRunes {
return nil, fmt.Errorf(
"document at index %d is %d characters; it does not fit beside a %d-character query "+
"in the %d-character request budget",
i, sizes[i], fixedRunes, limits.MaxTotalRunes,
)
}
}
if len(items) == 0 {
return nil, nil
}
if limits.MaxItems <= 0 && limits.MaxTotalRunes <= 0 {
return []Batch{{Start: 0, Items: items}}, nil
}
batches := make([]Batch, 0, 1)
start, total := 0, fixedRunes
for i := range items {
full := limits.MaxItems > 0 && i-start == limits.MaxItems
over := limits.MaxTotalRunes > 0 && total+sizes[i] > limits.MaxTotalRunes
if i > start && (full || over) {
batches = append(batches, Batch{Start: start, Items: items[start:i]})
start, total = i, fixedRunes
}
total += sizes[i]
}
batches = append(batches, Batch{Start: start, Items: items[start:]})
return batches, nil
}

View File

@ -0,0 +1,74 @@
package api
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSplitBatchesNoLimitsKeepsOneRequest(t *testing.T) {
got, err := SplitBatches([]string{"a", "b", "c"}, 0, BatchLimits{})
require.NoError(t, err)
require.Len(t, got, 1)
assert.Equal(t, 0, got[0].Start)
assert.Len(t, got[0].Items, 3)
}
func TestSplitBatchesSplitsOnItemCount(t *testing.T) {
items := []string{"a", "b", "c", "d", "e"}
got, err := SplitBatches(items, 0, BatchLimits{MaxItems: 2})
require.NoError(t, err)
require.Len(t, got, 3)
assert.Equal(t, []Batch{
{Start: 0, Items: items[0:2]},
{Start: 2, Items: items[2:4]},
{Start: 4, Items: items[4:5]},
}, got)
}
// The query travels in every request, so it is charged to every batch rather
// than once to the whole call.
func TestSplitBatchesChargesTheQueryToEveryBatch(t *testing.T) {
items := []string{strings.Repeat("x", 30), strings.Repeat("y", 30), strings.Repeat("z", 30)}
got, err := SplitBatches(items, 40, BatchLimits{MaxTotalRunes: 100})
require.NoError(t, err)
// 40 + 30 + 30 = 100 fits; adding the third would reach 130.
require.Len(t, got, 2)
assert.Len(t, got[0].Items, 2)
assert.Equal(t, 2, got[1].Start)
}
func TestSplitBatchesRejectsAnItemThatCannotFit(t *testing.T) {
_, err := SplitBatches([]string{"ok", strings.Repeat("x", 50)}, 0, BatchLimits{MaxItemRunes: 10})
require.Error(t, err)
assert.Contains(t, err.Error(), "index 1")
}
// A document that fits the per-document limit but not beside the query is
// still unsendable; saying so beats looping forever or silently dropping it.
func TestSplitBatchesRejectsAnItemThatCannotFitBesideTheQuery(t *testing.T) {
_, err := SplitBatches([]string{strings.Repeat("x", 80)}, 40, BatchLimits{MaxTotalRunes: 100})
require.Error(t, err)
assert.Contains(t, err.Error(), "does not fit beside")
}
func TestSplitBatchesRejectsAQueryThatFillsTheBudget(t *testing.T) {
_, err := SplitBatches([]string{"a"}, 100, BatchLimits{MaxTotalRunes: 100})
require.Error(t, err)
assert.Contains(t, err.Error(), "must also fit a document")
}
// Runes, not bytes: a CJK document is counted the way the vendor counts it.
func TestSplitBatchesCountsRunesNotBytes(t *testing.T) {
got, err := SplitBatches([]string{strings.Repeat("中", 10)}, 0, BatchLimits{MaxItemRunes: 10})
require.NoError(t, err)
assert.Len(t, got, 1)
}
func TestSplitBatchesEmptyInput(t *testing.T) {
got, err := SplitBatches(nil, 0, BatchLimits{MaxItems: 5})
require.NoError(t, err)
assert.Empty(t, got)
}

View File

@ -0,0 +1,167 @@
// Package cohererank implements the rerank shape Cohere defined and almost
// every other vendor copied: POST {base}/rerank with {model, query,
// documents} answering {results: [{index, relevance_score, document}]}.
//
// Jina, Zhipu, SiliconFlow, Qianfan, GPUStack, WeKnora Cloud and any
// OpenAI-compatible gateway that serves rerank all speak it. Their
// differences — which optional fields are accepted, where the endpoint sits,
// what the documented ceilings are — arrive as catalog.RerankSettings; this
// package contains no vendor names.
package cohererank
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
)
// Config is everything the client needs, already resolved by the catalog.
type Config struct {
Endpoint api.Endpoint
Settings catalog.RerankSettings
}
// Client talks the Cohere rerank shape to one endpoint.
type Client struct {
cfg Config
}
// New builds a client.
func New(cfg Config) *Client { return &Client{cfg: cfg} }
const defaultPath = "/rerank"
// url resolves the endpoint. A vendor whose default base URL already names
// the full rerank endpoint — and an operator who pasted one — must not have
// the path appended twice.
func (c *Client) url() string {
if c.cfg.Endpoint.URL != "" {
return c.cfg.Endpoint.Resolve("")
}
path := c.cfg.Settings.Path
if path == "" {
path = defaultPath
}
if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
path = ""
}
return c.cfg.Endpoint.Resolve(path)
}
type request struct {
Model string `json:"model"`
Query string `json:"query"`
Documents []string `json:"documents"`
TopN int `json:"top_n,omitempty"`
// TruncatePromptTokens is a vLLM extension, not part of any vendor's
// documented schema. It is sent only when the operator opted in, because
// a gateway that does not implement it rejects the unknown field.
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"`
ReturnDocuments *bool `json:"return_documents,omitempty"`
}
type response struct {
Results []result `json:"results"`
}
type result struct {
Index int `json:"index"`
RelevanceScore *float64 `json:"relevance_score"`
Score *float64 `json:"score"`
Document document `json:"document"`
}
// document is the echoed text, which vendors spell either as a bare string
// or as an object with a text field.
type document struct {
Text string
}
func (d *document) UnmarshalJSON(data []byte) error {
var text string
if err := json.Unmarshal(data, &text); err == nil {
d.Text = text
return nil
}
var obj struct {
Text string `json:"text"`
}
if err := json.Unmarshal(data, &obj); err != nil {
return fmt.Errorf("decode document: %w", err)
}
d.Text = obj.Text
return nil
}
// BuildRequestBody is the golden-test entry point: it returns the exact JSON
// object that would be sent.
func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) {
body := request{
Model: c.cfg.Endpoint.Model,
Query: query,
Documents: documents,
}
if c.cfg.Settings.AcceptsTruncatePromptTokens {
body.TruncatePromptTokens = c.cfg.Settings.TruncatePromptTokens
}
if c.cfg.Settings.SendTopN {
body.TopN = len(documents)
}
if c.cfg.Settings.SendReturnDocs {
enabled := true
body.ReturnDocuments = &enabled
}
raw, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
for k, v := range c.cfg.Settings.ExtraBody {
if _, exists := out[k]; !exists {
out[k] = v
}
}
return out, nil
}
// Rerank scores documents against the query, in the order they were given.
func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) {
body, err := c.BuildRequestBody(query, documents)
if err != nil {
return nil, err
}
var decoded response
if err := c.cfg.Endpoint.PostJSON(ctx, c.url(), body, &decoded); err != nil {
return nil, err
}
out := make([]api.RerankResult, 0, len(decoded.Results))
for _, item := range decoded.Results {
if item.Index < 0 || item.Index >= len(documents) {
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents))
}
out = append(out, api.RerankResult{
Index: item.Index,
Score: firstNonNil(item.RelevanceScore, item.Score),
Text: item.Document.Text,
})
}
return out, nil
}
// firstNonNil picks relevance_score, falling back to score: the field is
// spelled both ways across vendors that otherwise share this shape.
func firstNonNil(values ...*float64) float64 {
for _, v := range values {
if v != nil {
return *v
}
}
return 0
}

View File

@ -0,0 +1,203 @@
package cohererank
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// zhipuResponse is the documented response of Zhipu's rerank endpoint, whose
// result objects carry document / index / relevance_score and whose envelope
// adds id, request_id, created and usage:
// https://docs.bigmodel.cn/api-reference/模型-api/文本重排序
const zhipuResponse = `{
"id": "8305283656248111323",
"request_id": "8305283656248111323",
"created": 1769040000,
"results": [
{"index": 1, "relevance_score": 0.9819, "document": "上海气候"},
{"index": 0, "relevance_score": 0.0021, "document": "北京美食"}
],
"usage": {"prompt_tokens": 24, "total_tokens": 24}
}`
// jinaResponse spells the echoed document as an object instead of a string,
// which is the other shape vendors use for the same field:
// https://jina.ai/reranker/
const jinaResponse = `{
"model": "jina-reranker-v3.5",
"results": [
{"index": 0, "relevance_score": 0.95, "document": {"text": "上海气候"}},
{"index": 1, "relevance_score": 0.11, "document": {"text": "北京美食"}}
],
"usage": {"total_tokens": 32}
}`
func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client {
t.Helper()
return New(Config{
Endpoint: api.Endpoint{BaseURL: url, Model: "rerank", Auth: api.BearerAuth("k")},
Settings: settings,
})
}
func serve(t *testing.T, payload string) (*httptest.Server, *string, *map[string]any) {
t.Helper()
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
path := new(string)
body := new(map[string]any)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
*path = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(body)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(payload))
}))
return server, path, body
}
func TestRequestBodyCarriesOnlyWhatTheVendorDeclares(t *testing.T) {
c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{})
body, err := c.BuildRequestBody("上海天气", []string{"上海气候", "北京美食"})
require.NoError(t, err)
assert.Equal(t, map[string]any{
"model": "rerank",
"query": "上海天气",
"documents": []any{"上海气候", "北京美食"},
}, body, "optional fields must be absent unless the vendor asked for them")
}
func TestRequestBodyAddsTheOptionalFieldsAVendorDeclares(t *testing.T) {
c := newClient(t, "https://example.invalid/v1",
catalog.RerankSettings{SendTopN: true, SendReturnDocs: true})
body, err := c.BuildRequestBody("q", []string{"a", "b", "c"})
require.NoError(t, err)
assert.Equal(t, float64(3), body["top_n"], "top_n asks for every document back")
assert.Equal(t, true, body["return_documents"])
}
func TestDecodesAStringDocument(t *testing.T) {
server, _, _ := serve(t, zhipuResponse)
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
got, err := c.Rerank(context.Background(), "上海天气", []string{"北京美食", "上海气候"})
require.NoError(t, err)
assert.Equal(t, []api.RerankResult{
{Index: 1, Score: 0.9819, Text: "上海气候"},
{Index: 0, Score: 0.0021, Text: "北京美食"},
}, got)
}
func TestDecodesAnObjectDocument(t *testing.T) {
server, _, _ := serve(t, jinaResponse)
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
got, err := c.Rerank(context.Background(), "上海天气", []string{"上海气候", "北京美食"})
require.NoError(t, err)
assert.Equal(t, []api.RerankResult{
{Index: 0, Score: 0.95, Text: "上海气候"},
{Index: 1, Score: 0.11, Text: "北京美食"},
}, got)
}
// Some gateways serving this shape spell the score `score`. The pre-catalog
// client accepted both and so does this one.
func TestDecodesTheScoreAlias(t *testing.T) {
server, _, _ := serve(t, `{"results":[{"index":0,"score":0.42}]}`)
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
got, err := c.Rerank(context.Background(), "q", []string{"d"})
require.NoError(t, err)
require.Len(t, got, 1)
assert.InDelta(t, 0.42, got[0].Score, 1e-9)
}
func TestAppendsTheRerankPathToABareBaseURL(t *testing.T) {
server, path, _ := serve(t, `{"results":[]}`)
defer server.Close()
c := newClient(t, server.URL+"/v1", catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.NoError(t, err)
assert.Equal(t, "/v1/rerank", *path)
}
// Zhipu's default base URL already ends in /rerank, and operators paste full
// endpoints too. Appending the path again would 404.
func TestDoesNotDoubleTheRerankPath(t *testing.T) {
server, path, _ := serve(t, `{"results":[]}`)
defer server.Close()
c := newClient(t, server.URL+"/api/paas/v4/rerank", catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.NoError(t, err)
assert.Equal(t, "/api/paas/v4/rerank", *path)
}
// WeKnora Cloud serves the same shape on its own path.
func TestHonoursAVendorDeclaredPath(t *testing.T) {
server, path, _ := serve(t, `{"results":[]}`)
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{Path: "/api/v1/rerank"})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.NoError(t, err)
assert.Equal(t, "/api/v1/rerank", *path)
}
func TestRejectsAnOutOfRangeIndex(t *testing.T) {
server, _, _ := serve(t, `{"results":[{"index":9,"relevance_score":0.5}]}`)
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.Error(t, err)
assert.Contains(t, err.Error(), "out of range")
}
func TestSurfacesTheVendorErrorBody(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusUnprocessableEntity)
_, _ = w.Write([]byte(`{"error":{"message":"documents exceeds the limit"}}`))
}))
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.Error(t, err)
assert.Contains(t, err.Error(), "documents exceeds the limit")
}
// truncate_prompt_tokens is a vLLM extension that appears in no managed
// vendor's documented schema. It is sent only where the vendor declared it
// and the operator opted in, and never by default — sending it unasked is how
// an undocumented field ends up on every request to every gateway.
func TestTruncatePromptTokensIsAbsentByDefault(t *testing.T) {
c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{})
body, err := c.BuildRequestBody("q", []string{"d"})
require.NoError(t, err)
assert.NotContains(t, body, "truncate_prompt_tokens")
}
func TestTruncatePromptTokensIsSentWhenTheRowOptedIn(t *testing.T) {
c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{
AcceptsTruncatePromptTokens: true, TruncatePromptTokens: 512,
})
body, err := c.BuildRequestBody("q", []string{"d"})
require.NoError(t, err)
assert.Equal(t, float64(512), body["truncate_prompt_tokens"])
}

View File

@ -0,0 +1,116 @@
// Package dashscoperank implements Alibaba Model Studio's native text-rerank
// shape: the same fields as the Cohere dialect, wrapped in input/parameters
// and answered under output.
//
// https://help.aliyun.com/zh/model-studio/text-rerank-api
package dashscoperank
import (
"context"
"encoding/json"
"fmt"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
)
// Config is everything the client needs, already resolved by the catalog.
type Config struct {
Endpoint api.Endpoint
Settings catalog.RerankSettings
}
// Client talks DashScope's native rerank to one endpoint.
type Client struct {
cfg Config
}
// New builds a client.
func New(cfg Config) *Client { return &Client{cfg: cfg} }
type request struct {
Model string `json:"model"`
Input input `json:"input"`
Parameters parameters `json:"parameters"`
}
type input struct {
Query string `json:"query"`
Documents []string `json:"documents"`
}
type parameters struct {
ReturnDocuments bool `json:"return_documents"`
TopN int `json:"top_n"`
}
type response struct {
Output struct {
Results []struct {
Index int `json:"index"`
RelevanceScore float64 `json:"relevance_score"`
Document struct {
Text string `json:"text"`
} `json:"document"`
} `json:"results"`
} `json:"output"`
// DashScope reports failures in the body with a 200 on some paths.
Code string `json:"code"`
Message string `json:"message"`
}
// BuildRequestBody is the golden-test entry point.
func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) {
body := request{
Model: c.cfg.Endpoint.Model,
Input: input{Query: query, Documents: documents},
Parameters: parameters{
ReturnDocuments: c.cfg.Settings.SendReturnDocs,
// top_n is not optional in this shape: leaving it at zero would
// ask for no documents at all, so the full count is always sent.
TopN: len(documents),
},
}
raw, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
for k, v := range c.cfg.Settings.ExtraBody {
if _, exists := out[k]; !exists {
out[k] = v
}
}
return out, nil
}
// Rerank scores documents against the query.
func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) {
body, err := c.BuildRequestBody(query, documents)
if err != nil {
return nil, err
}
var decoded response
url := c.cfg.Endpoint.Resolve(c.cfg.Settings.Path)
if err := c.cfg.Endpoint.PostJSON(ctx, url, body, &decoded); err != nil {
return nil, err
}
if decoded.Code != "" {
return nil, fmt.Errorf("DashScope rerank error %s: %s", decoded.Code, decoded.Message)
}
out := make([]api.RerankResult, 0, len(decoded.Output.Results))
for _, item := range decoded.Output.Results {
if item.Index < 0 || item.Index >= len(documents) {
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents))
}
out = append(out, api.RerankResult{
Index: item.Index,
Score: item.RelevanceScore,
Text: item.Document.Text,
})
}
return out, nil
}

View File

@ -0,0 +1,116 @@
package dashscoperank
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// documentedResponse is the response shape of DashScope's text-rerank API,
// copied from the reference so the decoder is pinned against the vendor's own
// document: https://help.aliyun.com/zh/model-studio/text-rerank-api
const documentedResponse = `{
"output": {
"results": [
{"document": {"text": "上海气候"}, "index": 1, "relevance_score": 0.7314},
{"document": {"text": "北京美食"}, "index": 0, "relevance_score": 0.0002}
]
},
"usage": {"total_tokens": 24},
"request_id": "4c5b1c4d-0e2f-9b1a-8a1e-1b2c3d4e5f60"
}`
const endpointURL = "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client {
t.Helper()
return New(Config{
Endpoint: api.Endpoint{BaseURL: url, Model: "gte-rerank-v2", Auth: api.BearerAuth("k")},
Settings: settings,
})
}
func TestRequestBodyMatchesTheDocumentedSchema(t *testing.T) {
c := newClient(t, endpointURL, catalog.RerankSettings{SendReturnDocs: true})
body, err := c.BuildRequestBody("上海天气", []string{"北京美食", "上海气候"})
require.NoError(t, err)
assert.Equal(t, map[string]any{
"model": "gte-rerank-v2",
"input": map[string]any{
"query": "上海天气",
"documents": []any{"北京美食", "上海气候"},
},
"parameters": map[string]any{
"return_documents": true,
"top_n": float64(2),
},
}, body)
}
// top_n is mandatory in this shape: zero would ask for nothing back, so the
// full document count is always sent regardless of vendor settings.
func TestTopNAlwaysCarriesTheDocumentCount(t *testing.T) {
c := newClient(t, endpointURL, catalog.RerankSettings{})
body, err := c.BuildRequestBody("q", []string{"a", "b", "c", "d"})
require.NoError(t, err)
assert.Equal(t, float64(4), body["parameters"].(map[string]any)["top_n"])
}
func TestDecodesTheDocumentedResponse(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(documentedResponse))
}))
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
got, err := c.Rerank(context.Background(), "上海天气", []string{"北京美食", "上海气候"})
require.NoError(t, err)
assert.Equal(t, []api.RerankResult{
{Index: 1, Score: 0.7314, Text: "上海气候"},
{Index: 0, Score: 0.0002, Text: "北京美食"},
}, got)
}
// DashScope answers some failures with a 200 and an error code in the body,
// which would otherwise decode as an empty result set.
func TestSurfacesAnInBodyError(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"code":"InvalidParameter","message":"documents is too long","request_id":"r"}`))
}))
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.Error(t, err)
assert.Contains(t, err.Error(), "documents is too long")
}
func TestPostsToTheBaseURLItself(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
var gotPath string
var gotBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&gotBody)
_, _ = w.Write([]byte(`{"output":{"results":[]}}`))
}))
defer server.Close()
c := newClient(t, server.URL+"/api/v1/services/rerank/text-rerank/text-rerank", catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d"})
require.NoError(t, err)
assert.Equal(t, "/api/v1/services/rerank/text-rerank/text-rerank", gotPath,
"the vendor's base URL already names the full endpoint")
}

View File

@ -135,6 +135,40 @@ func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
return resp, nil
}
// PostJSON sends one authenticated JSON request and decodes the reply into
// out. It is the whole round-trip for the non-streaming protocols (rerank,
// embeddings), which have no SSE to assemble.
//
// It never returns a nil error with nothing decoded: every failure path —
// marshalling, SSRF, transport, non-2xx, decoding — returns an error, so a
// caller cannot mistake an empty result for a successful empty answer.
func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error {
req, _, err := e.NewRequest(ctx, url, body, false)
if err != nil {
return err
}
// No LogRequest here: the chat protocols log their own bodies because a
// prompt is what an operator debugs, while a rerank body is a query plus
// every candidate chunk. The rerank layer logs a truncated line at Debug
// instead, so this would have duplicated it at Info.
resp, err := e.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
raw, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read response: %w", err)
}
if out == nil {
return nil
}
if err := json.Unmarshal(raw, out); err != nil {
return fmt.Errorf("decode response: %w", err)
}
return nil
}
// HTTPError is a non-2xx vendor reply.
type HTTPError struct {
StatusCode int

View File

@ -0,0 +1,106 @@
// Package nimrerank implements NVIDIA NIM's retrieval reranking shape: a
// query object, a passages array, and rankings carrying a logit.
//
// Two things separate it from the Cohere dialect and both matter to callers.
// The score is an unbounded log-odds value that is routinely negative, not a
// 0..1 relevance — catalog.RerankSettings.ScoreScale says so. And `truncate`
// defaults to "NONE", which means an over-long passage fails the request
// instead of being cut, so the vendor sets it explicitly.
//
// https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer
package nimrerank
import (
"context"
"encoding/json"
"fmt"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
)
// Config is everything the client needs, already resolved by the catalog.
type Config struct {
Endpoint api.Endpoint
Settings catalog.RerankSettings
}
// Client talks NIM reranking to one endpoint.
type Client struct {
cfg Config
}
// New builds a client.
func New(cfg Config) *Client { return &Client{cfg: cfg} }
type text struct {
Text string `json:"text"`
}
type request struct {
Model string `json:"model"`
Query text `json:"query"`
Passages []text `json:"passages"`
Truncate string `json:"truncate,omitempty"`
}
type response struct {
Rankings []struct {
Index int `json:"index"`
Logit float64 `json:"logit"`
} `json:"rankings"`
}
// BuildRequestBody is the golden-test entry point.
func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) {
passages := make([]text, len(documents))
for i := range documents {
passages[i] = text{Text: documents[i]}
}
body := request{
Model: c.cfg.Endpoint.Model,
Query: text{Text: query},
Passages: passages,
Truncate: c.cfg.Settings.Truncate,
}
raw, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
for k, v := range c.cfg.Settings.ExtraBody {
if _, exists := out[k]; !exists {
out[k] = v
}
}
return out, nil
}
// Rerank scores documents against the query. The returned scores are logits;
// see the package comment.
func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) {
body, err := c.BuildRequestBody(query, documents)
if err != nil {
return nil, err
}
var decoded response
url := c.cfg.Endpoint.Resolve(c.cfg.Settings.Path)
if err := c.cfg.Endpoint.PostJSON(ctx, url, body, &decoded); err != nil {
return nil, err
}
out := make([]api.RerankResult, 0, len(decoded.Rankings))
for _, item := range decoded.Rankings {
if item.Index < 0 || item.Index >= len(documents) {
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents))
}
out = append(out, api.RerankResult{
Index: item.Index,
Score: item.Logit,
Text: documents[item.Index],
})
}
return out, nil
}

View File

@ -0,0 +1,128 @@
package nimrerank
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// documentedResponse is the response example from NVIDIA's reranking
// reference, copied verbatim so the decoder is pinned against the vendor's
// own document rather than against our structs:
// https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer
//
// Note the negative logit. This protocol does not return a 0..1 relevance
// score, which is why catalog.RerankSettings.ScoreScale exists.
const documentedResponse = `{
"rankings": [
{"index": 2, "logit": 0.226318359375},
{"index": 1, "logit": -1.171875},
{"index": 0, "logit": -6.3125}
]
}`
func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client {
t.Helper()
return New(Config{
Endpoint: api.Endpoint{BaseURL: url, Model: "nvidia/nv-rerankqa-mistral-4b-v3", Auth: api.BearerAuth("k")},
Settings: settings,
})
}
func TestRequestBodyMatchesTheDocumentedSchema(t *testing.T) {
c := newClient(t, "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking",
catalog.RerankSettings{Truncate: "END", ScoreScale: api.ScoreLogit})
body, err := c.BuildRequestBody("q", []string{"d0", "d1"})
require.NoError(t, err)
assert.Equal(t, map[string]any{
"model": "nvidia/nv-rerankqa-mistral-4b-v3",
"query": map[string]any{"text": "q"},
"passages": []any{map[string]any{"text": "d0"}, map[string]any{"text": "d1"}},
"truncate": "END",
}, body)
}
// truncate defaults to NONE upstream, which fails the request on an over-long
// passage instead of cutting it. A vendor that does not set it must not have
// the key invented for it either.
func TestTruncateIsOmittedWhenUnset(t *testing.T) {
c := newClient(t, "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", catalog.RerankSettings{})
body, err := c.BuildRequestBody("q", []string{"d0"})
require.NoError(t, err)
assert.NotContains(t, body, "truncate")
}
func TestDecodesTheDocumentedResponse(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(documentedResponse))
}))
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{ScoreScale: api.ScoreLogit})
got, err := c.Rerank(context.Background(), "q", []string{"d0", "d1", "d2"})
require.NoError(t, err)
assert.Equal(t, []api.RerankResult{
{Index: 2, Score: 0.226318359375, Text: "d2"},
{Index: 1, Score: -1.171875, Text: "d1"},
{Index: 0, Score: -6.3125, Text: "d0"},
}, got)
}
// An index the request never sent is a protocol violation, not something to
// map onto whatever document happens to sit at that offset.
func TestRejectsAnOutOfRangeIndex(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"rankings":[{"index":7,"logit":0.5}]}`))
}))
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d0"})
require.Error(t, err)
assert.Contains(t, err.Error(), "out of range")
}
func TestSurfacesTheVendorErrorBody(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"detail":"input too long"}`))
}))
defer server.Close()
c := newClient(t, server.URL, catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d0"})
require.Error(t, err)
assert.Contains(t, err.Error(), "input too long")
}
func TestRequestReachesTheConfiguredURL(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
var gotPath string
var gotBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&gotBody)
_, _ = w.Write([]byte(`{"rankings":[]}`))
}))
defer server.Close()
c := newClient(t, server.URL+"/v1/retrieval/nvidia/reranking", catalog.RerankSettings{})
_, err := c.Rerank(context.Background(), "q", []string{"d0"})
require.NoError(t, err)
assert.Equal(t, "/v1/retrieval/nvidia/reranking", gotPath, "the base URL already names the endpoint")
assert.Equal(t, "q", gotBody["query"].(map[string]any)["text"])
}

View File

@ -0,0 +1,83 @@
package api
import "context"
// RerankAPI names a rerank wire protocol. It is deliberately a separate type
// from API, the chat protocol selector: the two vocabularies must not mix, or
// a chat row whose extra_config.api names a rerank dialect would pass
// validation and fail at call time.
type RerankAPI string
// The rerank protocols WeKnora speaks.
const (
// RerankCohere is the de-facto standard shape — POST {base}/rerank with
// {model, query, documents} and a results array of {index,
// relevance_score, document}. Cohere defined it; Jina, Zhipu,
// SiliconFlow, Qianfan, GPUStack and every OpenAI-compatible gateway
// that serves rerank at all copy it.
RerankCohere RerankAPI = "cohere-rerank"
// RerankDashScope is Alibaba Model Studio's native shape, which wraps
// the same fields in input/parameters and answers under output.
RerankDashScope RerankAPI = "dashscope-rerank"
// RerankNIM is NVIDIA NIM's retrieval shape: a query object, a passages
// array, and rankings carrying an unbounded logit instead of a
// probability.
RerankNIM RerankAPI = "nim-rerank"
// RerankTencentLKEAP is Tencent Cloud's RunRerank action, reached through
// the official SDK because it is TC3-signed rather than key-authenticated.
RerankTencentLKEAP RerankAPI = "tencent-lkeap"
// RerankVolcengineKnowledge is Volcengine's managed Knowledge Service
// rerank, reached through the vikingdb SDK for the same reason.
RerankVolcengineKnowledge RerankAPI = "volcengine-knowledge"
)
// Known reports whether the value names a protocol this build implements.
func (a RerankAPI) Known() bool {
switch a {
case RerankCohere, RerankDashScope, RerankNIM,
RerankTencentLKEAP, RerankVolcengineKnowledge:
return true
}
return false
}
// HTTPServed reports whether a protocol package implements the protocol. The
// SDK-backed ones are built by the rerank package itself, which is where the
// vendor SDKs are imported.
func (a RerankAPI) HTTPServed() bool {
switch a {
case RerankCohere, RerankDashScope, RerankNIM:
return true
}
return false
}
// ScoreScale describes what a rerank score means, which differs per protocol
// and cannot be inferred from the number itself.
type ScoreScale string
const (
// ScoreProbability is a 0..1 relevance score. Every protocol but NIM
// returns one.
ScoreProbability ScoreScale = "probability"
// ScoreLogit is an unbounded log-odds value that may be negative. NIM
// returns one, so a threshold tuned for probabilities rejects almost
// everything on that vendor.
ScoreLogit ScoreScale = "logit"
)
// RerankResult is one scored document, in the vendor's own score scale.
// Index refers to the position in the documents slice that was sent.
type RerankResult struct {
Index int
Score float64
// Text is the document echoed back, empty when the vendor was not asked
// to return documents. Callers index into their own slice instead of
// relying on it.
Text string
}
// Reranker is what every rerank protocol package implements.
type Reranker interface {
Rerank(ctx context.Context, query string, documents []string) ([]RerankResult, error)
}

View File

@ -341,6 +341,9 @@ type VendorCompat struct {
OpenAIResponses OpenAIResponsesCompat `json:"openai_responses,omitempty"`
AnthropicMessages AnthropicMessagesCompat `json:"anthropic_messages,omitempty"`
GoogleGenerativeAI GoogleGenerativeAICompat `json:"google_generative_ai,omitempty"`
// Rerank is the rerank protocol overlay. It has no per-protocol variants:
// a vendor serves exactly one rerank dialect.
Rerank RerankCompat `json:"rerank,omitempty"`
}
// apply writes the set fields of an overlay struct onto a settings struct

View File

@ -194,7 +194,14 @@ func TestLegacyRowsValidateAndResolve(t *testing.T) {
if resolved.Vendor == nil {
t.Fatal("resolved with a nil vendor")
}
if !resolved.API.Known() {
// Rerank rows resolve to a rerank protocol; everything else
// resolves to a chat one. The two vocabularies are separate
// types so a row can never land on the wrong one.
if row.typ == types.ModelTypeRerank {
if !resolved.RerankAPI.Known() {
t.Fatalf("resolved to an unknown rerank protocol %q", resolved.RerankAPI)
}
} else if !resolved.API.Known() {
t.Fatalf("resolved to an unknown protocol %q", resolved.API)
}
if resolved.RemoteModel == "" && row.model != "" {

View File

@ -31,6 +31,9 @@ func Register(v *Vendor) {
if v.Auth == "" {
v.Auth = AuthBearer
}
if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) {
v.RerankAPI = api.RerankCohere
}
for i := range v.Models {
if v.Models[i].Type == "" {
v.Models[i].Type = types.ModelTypeKnowledgeQA

View File

@ -0,0 +1,92 @@
package catalog
import "github.com/Tencent/WeKnora/internal/models/api"
// RerankCompat is the overlay form (every field optional) of the rerank
// settings. JSON keys are the documented names used in models.json and
// config/models.json.
type RerankCompat struct {
Path *string `json:"path,omitempty"`
SendTopN *bool `json:"send_top_n,omitempty"`
SendReturnDocs *bool `json:"send_return_documents,omitempty"`
ScoreScale *api.ScoreScale `json:"score_scale,omitempty"`
Truncate *string `json:"truncate,omitempty"`
MaxDocuments *int `json:"max_documents,omitempty"`
MaxQueryChars *int `json:"max_query_chars,omitempty"`
MaxDocumentChars *int `json:"max_document_chars,omitempty"`
MaxRequestChars *int `json:"max_request_chars,omitempty"`
MaxConcurrency *int `json:"max_concurrency,omitempty"`
RequestTimeout *int `json:"request_timeout_seconds,omitempty"`
// AcceptsTruncatePromptTokens marks a vLLM-class runtime.
AcceptsTruncatePromptTokens *bool `json:"accepts_truncate_prompt_tokens,omitempty"`
// UnsupportedReason marks a model this build cannot call.
UnsupportedReason *string `json:"unsupported_reason,omitempty"`
ExtraBody map[string]any `json:"extra_body,omitempty"`
}
// RerankSettings is the resolved (fully defaulted) form.
type RerankSettings struct {
// Path is appended to the base URL by protocols that use one. Vendors
// whose default base URL already names the full endpoint leave it empty.
Path string
// SendTopN asks for every document back by naming their count. Omitting
// it means "all", which is what every protocol here does by default.
SendTopN bool
// SendReturnDocs asks the vendor to echo the document text. Callers index
// into their own slice, so this is off unless a vendor needs it.
SendReturnDocs bool
// ScoreScale says what the returned numbers mean. See api.ScoreScale:
// getting this wrong makes a relevance threshold meaningless.
ScoreScale api.ScoreScale
// Truncate is the vendor's server-side truncation setting for
// over-long input ("END" on NIM). Empty sends nothing.
Truncate string
// Limits carry the documented per-request ceilings. MaxQueryChars is
// checked on its own rather than through BatchLimits: the query is not an
// item to be split, so an over-long one cannot be made to fit by batching.
MaxDocuments int
MaxQueryChars int
MaxDocumentChars int
MaxRequestChars int
// MaxConcurrency bounds in-flight batches when the input is split. 0
// means the caller's default.
MaxConcurrency int
// RequestTimeout caps one request, in seconds. 0 leaves the HTTP client
// without its own deadline and lets the caller's context govern, which is
// what every vendor but WeKnora Cloud has always done here.
RequestTimeout int
// AcceptsTruncatePromptTokens reports that this vendor is a vLLM-class
// runtime, which is the only kind that implements the
// `truncate_prompt_tokens` extension. It is not part of the Cohere shape
// and appears in no managed vendor's schema, so it must never be sent to
// one: that is how an undocumented field ends up on every request.
AcceptsTruncatePromptTokens bool
// UnsupportedReason is set on a catalog entry naming a rerank dialect no
// protocol package implements. Resolve refuses it rather than letting a
// request go out shaped for a different protocol, which surfaces as a
// decode error far from its cause.
UnsupportedReason string
// TruncatePromptTokens is the budget the operator opted into on this row,
// honoured only where AcceptsTruncatePromptTokens is set. It is a row
// setting rather than a vendor fact because one runtime serves models
// with different context windows. Zero sends nothing.
TruncatePromptTokens int
ExtraBody map[string]any
}
// BatchLimits renders the documented ceilings for api.SplitBatches.
func (s RerankSettings) BatchLimits() api.BatchLimits {
return api.BatchLimits{
MaxItems: s.MaxDocuments,
MaxItemRunes: s.MaxDocumentChars,
MaxTotalRunes: s.MaxRequestChars,
}
}
// DefaultRerank is the protocol baseline: ask for every document, expect a
// 0..1 relevance score, enforce no ceiling the vendor did not state.
func DefaultRerank() RerankSettings {
return RerankSettings{
ScoreScale: api.ScoreProbability,
}
}

View File

@ -3,6 +3,7 @@ package catalog
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/Tencent/WeKnora/internal/models/api"
@ -21,6 +22,16 @@ const (
// ExtraThinkingControl is the legacy thinking encoding selector written
// by older UIs: none | enable_thinking | thinking_type | chat_template_kwargs.
ExtraThinkingControl = "thinking_control"
// ExtraTruncatePromptTokens is the vLLM-only server-side truncation
// budget for rerank, opt-in per row. It is never sent unless the operator
// set it: vendors that do not implement it reject the unknown field.
ExtraTruncatePromptTokens = "truncate_prompt_tokens"
// ExtraScoreScale overrides the vendor rerank score scale for one row.
// A self-hosted gateway serves whatever reranker was deployed behind it
// and the two families disagree: BGE-class models answer a 0..1
// probability, Qwen3-Reranker-class models an unbounded score. A vendor
// can only state what its own documentation shows.
ExtraScoreScale = "score_scale"
)
// Ref identifies one configured model.
@ -51,6 +62,12 @@ type Resolved struct {
OpenAIResponses OpenAIResponsesSettings
AnthropicMessages AnthropicMessagesSettings
GoogleGenerativeAI GoogleGenerativeAISettings
// RerankAPI and Rerank are filled only when the reference asked for a
// rerank model. Chat resolution is on every request's hot path, so the
// rerank overlay is not merged for it.
RerankAPI api.RerankAPI
Rerank RerankSettings
}
// Resolve merges the vendor, catalog entry, extra-config and per-row
@ -86,6 +103,10 @@ func Resolve(ref Ref) (*Resolved, error) {
applySpecOverride(&spec, ref.Override)
}
if modelType == types.ModelTypeRerank {
return resolveRerank(ref, vendor, spec, cataloged, baseURL)
}
resolvedAPI := spec.API
if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" {
resolvedAPI = api.API(forced)
@ -346,3 +367,84 @@ func (r *Resolved) Capabilities() Capabilities {
caps.ThinkingLevels = r.ThinkingLevels.SupportedLevels()
return caps
}
// resolveRerank merges the rerank layers. Rerank has one settings struct
// rather than one per protocol, so the model entry's compat object is decoded
// unconditionally instead of being matched against a protocol.
func resolveRerank(
ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string,
) (*Resolved, error) {
protocol := vendor.RerankAPI
if protocol == "" {
protocol = api.RerankCohere
}
if !protocol.Known() {
return nil, fmt.Errorf("catalog: unknown rerank api %q for provider %s", protocol, vendor.ID)
}
// Lowest precedence first: protocol default, vendor, model entry, row spec
// override. extra_config is the operator speaking about this one row, so it
// comes last.
settings := DefaultRerank()
apply(&settings, &vendor.Compat.Rerank)
if len(spec.Compat) > 0 {
overlay := &RerankCompat{}
if err := decodeCompat(spec.Compat, overlay); err != nil {
return nil, fmt.Errorf("rerank compat: %w", err)
}
apply(&settings, overlay)
}
if raw := ref.Override.CompatJSON(); len(raw) > 0 {
overlay := &RerankCompat{}
if err := decodeCompat(raw, overlay); err != nil {
return nil, fmt.Errorf("rerank compat: %w", err)
}
apply(&settings, overlay)
}
if raw := strings.TrimSpace(ref.Extra[ExtraScoreScale]); raw != "" {
scale := api.ScoreScale(strings.ToLower(raw))
if scale != api.ScoreProbability && scale != api.ScoreLogit {
return nil, fmt.Errorf(
"catalog: invalid %s in extra_config: %q (expected %q or %q)",
ExtraScoreScale, raw, api.ScoreProbability, api.ScoreLogit,
)
}
settings.ScoreScale = scale
}
if raw := strings.TrimSpace(ref.Extra[ExtraTruncatePromptTokens]); raw != "" {
if !settings.AcceptsTruncatePromptTokens {
return nil, fmt.Errorf(
"catalog: %s is a vLLM extension and %s does not implement it; "+
"remove it from extra_config (it is accepted by self-hosted runtimes only)",
ExtraTruncatePromptTokens, vendor.ID,
)
}
budget, err := strconv.Atoi(raw)
if err != nil || budget <= 0 {
return nil, fmt.Errorf("catalog: invalid %s in extra_config: %q", ExtraTruncatePromptTokens, raw)
}
settings.TruncatePromptTokens = budget
}
// Checked after every layer: a model entry is what declares a dialect
// this build cannot speak.
if settings.UnsupportedReason != "" {
return nil, fmt.Errorf(
"catalog: %s does not serve %q through a protocol this build implements: %s",
vendor.ID, spec.ID, settings.UnsupportedReason,
)
}
out := &Resolved{
Vendor: vendor,
Spec: spec,
Cataloged: cataloged,
BaseURL: baseURL,
RemoteModel: ref.Model,
RerankAPI: protocol,
Rerank: settings,
}
if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" {
out.RemoteModel = override
}
return out, nil
}

View File

@ -42,14 +42,16 @@ const (
// ExtraField describes one vendor-specific configuration input the model
// editor renders dynamically (Azure api-version, LKEAP secret key, ...).
type ExtraField struct {
Key string `json:"key"`
Label string `json:"label"`
Labels map[string]string `json:"labels,omitempty"`
Type string `json:"type"` // "string", "number", "boolean", "select", "password"
Required bool `json:"required"`
Default string `json:"default,omitempty"`
Placeholder string `json:"placeholder,omitempty"`
Options []ExtraFieldOption `json:"options,omitempty"`
Key string `json:"key"`
Label string `json:"label"`
Labels map[string]string `json:"labels,omitempty"`
Type string `json:"type"` // "string", "number", "boolean", "select", "password"
Required bool `json:"required"`
Default string `json:"default,omitempty"`
Placeholder string `json:"placeholder,omitempty"`
// Placeholders carries localized variants of Placeholder.
Placeholders map[string]string `json:"placeholders,omitempty"`
Options []ExtraFieldOption `json:"options,omitempty"`
// ModelTypes restricts the field to some model types; empty means all.
ModelTypes []types.ModelType `json:"model_types,omitempty"`
// Secret marks values that must never be echoed back to the UI.
@ -102,6 +104,11 @@ func localizedOr(table map[string]string, locale, fallback string) string {
return fallback
}
// LocalizedPlaceholder resolves the placeholder for a locale.
func (f ExtraField) LocalizedPlaceholder(locale string) string {
return localizedOr(f.Placeholders, locale, f.Placeholder)
}
// CredentialLabelFor returns the credential naming for a model type, or nil
// when the vendor uses the generic API-key wording.
func (v *Vendor) CredentialLabelFor(modelType types.ModelType) *CredentialLabel {
@ -119,10 +126,19 @@ func (v *Vendor) CredentialLabelFor(modelType types.ModelType) *CredentialLabel
return nil
}
// ExtraFieldOption is one choice of a select field.
// ExtraFieldOption is one choice of a select field. Labels carries localized
// variants keyed by locale, like every other operator-facing string here: an
// option whose label is prose rather than an identifier is unreadable to half
// the product without it.
type ExtraFieldOption struct {
Label string `json:"label"`
Value string `json:"value"`
Label string `json:"label"`
Labels map[string]string `json:"labels,omitempty"`
Value string `json:"value"`
}
// LocalizedLabel resolves the option label for a locale.
func (o ExtraFieldOption) LocalizedLabel(locale string) string {
return localizedOr(o.Labels, locale, o.Label)
}
// ModelCost is priced per million tokens, in USD unless Currency says otherwise.
@ -223,6 +239,9 @@ type Vendor struct {
Icon []byte
// API is the default chat protocol; a ModelSpec may override it.
API api.API
// RerankAPI is the rerank protocol. Register defaults it to the Cohere
// shape for any vendor that serves rerank without naming another.
RerankAPI api.RerankAPI
// DefaultBaseURLs by model type; GetDefaultURL falls back to chat.
DefaultBaseURLs map[types.ModelType]string
ModelTypes []types.ModelType

View File

@ -16,6 +16,7 @@
package parity
import (
"bytes"
"context"
"encoding/json"
"net/http"
@ -523,6 +524,13 @@ func TestCatalogInvariants(t *testing.T) {
// Resolve validates the compat object against the model's
// protocol: an unknown or misspelled key fails here.
_, err := catalog.Resolve(catalog.Ref{Provider: v.ID, Model: m.ID, ModelType: m.Type})
if bytes.Contains(m.Compat, []byte("unsupported_reason")) {
// An entry that declares itself unserveable must say so
// by refusing, not by resolving into a request shaped
// for a protocol this build does not speak.
assert.Error(t, err, "%s: declares unsupported_reason but still resolves", label)
continue
}
assert.NoError(t, err, "%s: does not resolve", label)
}
}

View File

@ -0,0 +1,522 @@
package parity
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestEveryRerankVendorResolvesToAKnownProtocol walks the catalog, so a newly
// added vendor is covered without touching this file.
func TestEveryRerankVendorResolvesToAKnownProtocol(t *testing.T) {
vendors := catalog.ListByType(types.ModelTypeRerank)
require.NotEmpty(t, vendors, "no vendor serves rerank")
for _, v := range vendors {
t.Run(v.ID, func(t *testing.T) {
model := "some-rerank-model"
if catalogued := v.ModelsByType(types.ModelTypeRerank); len(catalogued) > 0 {
model = catalogued[0].ID
}
resolved, err := catalog.Resolve(catalog.Ref{
Provider: v.ID, Model: model, ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
assert.True(t, resolved.RerankAPI.Known(),
"unknown rerank protocol %q", resolved.RerankAPI)
// The catch-all vendor has no endpoint of its own: the operator
// supplies one, and Validate requires it.
if v.ID != catalog.GenericID {
assert.NotEmpty(t, resolved.BaseURL, "rerank vendors need a default endpoint")
}
// A vendor that declares a ceiling must declare a usable one.
assert.GreaterOrEqual(t, resolved.Rerank.MaxDocuments, 0)
assert.GreaterOrEqual(t, resolved.Rerank.MaxDocumentChars, 0)
if resolved.Rerank.ScoreScale != "" {
assert.Contains(t,
[]api.ScoreScale{api.ScoreProbability, api.ScoreLogit},
resolved.Rerank.ScoreScale)
}
})
}
}
// TestRerankProtocolAssignment pins which dialect each vendor speaks. The
// wire shapes are mutually incompatible, so a silent reassignment breaks
// every rerank call for that vendor.
func TestRerankProtocolAssignment(t *testing.T) {
for id, want := range map[string]api.RerankAPI{
"aliyun": api.RerankDashScope,
"nvidia": api.RerankNIM,
"lkeap": api.RerankTencentLKEAP,
"volcengine": api.RerankVolcengineKnowledge,
"zhipu": api.RerankCohere,
"jina": api.RerankCohere,
"siliconflow": api.RerankCohere,
"qianfan": api.RerankCohere,
"gpustack": api.RerankCohere,
"generic": api.RerankCohere,
"openai": api.RerankCohere,
"weknoracloud": api.RerankCohere,
"novita": api.RerankCohere,
"openrouter": api.RerankCohere,
"litellm": api.RerankCohere,
} {
t.Run(id, func(t *testing.T) {
resolved, err := catalog.Resolve(catalog.Ref{
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
assert.Equal(t, want, resolved.RerankAPI)
})
}
}
// TestRerankOutboundShapePerProtocol builds the real client through
// rerank.NewReranker and asserts what each dialect puts on the wire.
func TestRerankOutboundShapePerProtocol(t *testing.T) {
for _, tc := range []struct {
name string
provider string
model string
assert func(t *testing.T, path string, body map[string]any)
reply string
}{
{
name: "cohere shape", provider: "siliconflow", model: "BAAI/bge-reranker-v2-m3",
reply: `{"results":[{"index":0,"relevance_score":0.5}]}`,
assert: func(t *testing.T, path string, body map[string]any) {
assert.Equal(t, "/v1/rerank", path)
assert.Equal(t, "BAAI/bge-reranker-v2-m3", body["model"])
assert.Equal(t, "q", body["query"])
assert.Equal(t, []any{"d0"}, body["documents"])
},
},
{
name: "dashscope shape", provider: "aliyun", model: "gte-rerank-v2",
reply: `{"output":{"results":[{"index":0,"relevance_score":0.5,"document":{"text":"d0"}}]}}`,
assert: func(t *testing.T, _ string, body map[string]any) {
input, ok := body["input"].(map[string]any)
require.True(t, ok, "DashScope wraps the query and documents in input")
assert.Equal(t, "q", input["query"])
assert.Contains(t, body, "parameters")
assert.NotContains(t, body, "documents", "documents live under input")
},
},
{
name: "nim shape", provider: "nvidia", model: "nvidia/nv-rerankqa-mistral-4b-v3",
reply: `{"rankings":[{"index":0,"logit":0.0}]}`,
assert: func(t *testing.T, _ string, body map[string]any) {
query, ok := body["query"].(map[string]any)
require.True(t, ok, "NIM sends the query as an object")
assert.Equal(t, "q", query["text"])
assert.Equal(t, []any{map[string]any{"text": "d0"}}, body["passages"])
assert.Equal(t, "END", body["truncate"], "NIM fails over-long input unless told to cut")
},
},
} {
t.Run(tc.name, func(t *testing.T) {
var gotPath string
var gotBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&gotBody)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(tc.reply))
}))
defer server.Close()
reranker, err := rerank.NewReranker(&rerank.RerankerConfig{
Provider: tc.provider, ModelName: tc.model, APIKey: "k",
BaseURL: server.URL + rerankPathFor(t, tc.provider),
})
require.NoError(t, err)
_, err = reranker.Rerank(context.Background(), "q", []string{"d0"})
require.NoError(t, err)
tc.assert(t, gotPath, gotBody)
})
}
}
// rerankPathFor keeps the loopback base URL shaped like the vendor's own, so
// the test exercises the same path handling production does.
func rerankPathFor(t *testing.T, provider string) string {
t.Helper()
switch provider {
case "aliyun":
return "/api/v1/services/rerank/text-rerank/text-rerank"
case "nvidia":
return "/v1/retrieval/nvidia/reranking"
default:
return "/v1"
}
}
// TestNvidiaRerankScoresAreComparable is the end-to-end form of the unit test
// in the rerank package: a NIM logit must reach the retrieval pipeline on the
// same 0..1 scale as every other vendor, because they all meet one
// RerankThreshold.
func TestNvidiaRerankScoresAreComparable(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"rankings":[{"index":0,"logit":-1.171875},{"index":1,"logit":4.0}]}`))
}))
defer server.Close()
reranker, err := rerank.NewReranker(&rerank.RerankerConfig{
Provider: "nvidia", ModelName: "nvidia/nv-rerankqa-mistral-4b-v3", APIKey: "k",
BaseURL: server.URL + "/v1/retrieval/nvidia/reranking",
})
require.NoError(t, err)
got, err := reranker.Rerank(context.Background(), "q", []string{"d0", "d1"})
require.NoError(t, err)
require.Len(t, got, 2)
for _, item := range got {
assert.GreaterOrEqual(t, item.RelevanceScore, 0.0)
assert.LessOrEqual(t, item.RelevanceScore, 1.0)
}
// Results come back ranked, so the logit of 4.0 leads and the negative
// one follows: the conversion is monotonic and the ranking is preserved.
assert.Equal(t, 1, got[0].Index)
assert.Greater(t, got[0].RelevanceScore, got[1].RelevanceScore)
}
// TestSignedRerankVendorRequiresItsIdentityPair keeps the pre-catalog
// contract: WeKnora Cloud rerank is signed with an AppID / AppSecret pair, and
// a row without one must fail at construction rather than send unsigned
// requests that the far end rejects with something less readable.
func TestSignedRerankVendorRequiresItsIdentityPair(t *testing.T) {
_, err := rerank.NewReranker(&rerank.RerankerConfig{
Provider: "weknoracloud", ModelName: "rerank",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "AppID")
_, err = rerank.NewReranker(&rerank.RerankerConfig{
Provider: "weknoracloud", ModelName: "rerank", AppID: "app",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "AppSecret")
}
// TestTruncatePromptTokensOnlyReachesVLLMClassVendors pins the gate on the
// vLLM extension restored from the pre-catalog client. It is opt-in per row,
// but the row can only opt into it on a runtime that implements it: no
// managed vendor documents the field, and sending it to one is how an
// undocumented parameter ends up on every request.
func TestTruncatePromptTokensOnlyReachesVLLMClassVendors(t *testing.T) {
optIn := map[string]string{catalog.ExtraTruncatePromptTokens: "512"}
for _, id := range []string{"generic", "gpustack"} {
t.Run(id+" accepts it", func(t *testing.T) {
resolved, err := catalog.Resolve(catalog.Ref{
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
BaseURL: "http://127.0.0.1:9/v1", Extra: optIn,
})
require.NoError(t, err)
assert.Equal(t, 512, resolved.Rerank.TruncatePromptTokens)
})
}
for _, id := range []string{"jina", "zhipu", "siliconflow", "qianfan", "weknoracloud", "aliyun", "nvidia"} {
t.Run(id+" rejects it", func(t *testing.T) {
_, err := catalog.Resolve(catalog.Ref{
Provider: id, Model: "m", ModelType: types.ModelTypeRerank, Extra: optIn,
})
require.Error(t, err, "a managed vendor must not silently accept a vLLM extension")
assert.Contains(t, err.Error(), "vLLM extension")
})
}
}
func TestTruncatePromptTokensRejectsAnInvalidValue(t *testing.T) {
for _, raw := range []string{"0", "-1", "abc"} {
_, err := catalog.Resolve(catalog.Ref{
Provider: "generic", Model: "m", ModelType: types.ModelTypeRerank,
BaseURL: "http://127.0.0.1:9/v1",
Extra: map[string]string{catalog.ExtraTruncatePromptTokens: raw},
})
require.Error(t, err, "value %q", raw)
}
}
// TestRerankCeilingsAreTheDocumentedOnes pins the numbers the shared batching
// layer enforces. They used to live inside each client, where a dedicated
// test watched them; now they are declarations, so they need their own guard —
// TestEveryRerankVendorResolvesToAKnownProtocol only checks they are sane.
func TestRerankCeilingsAreTheDocumentedOnes(t *testing.T) {
for id, want := range map[string]catalog.RerankSettings{
// docs.bigmodel.cn: 最多 128 条,query 与单条文档各 4096 字符
"zhipu": {MaxDocuments: 128, MaxQueryChars: 4096, MaxDocumentChars: 4096},
// cloud.tencent.com/document/product/1772: RunRerank 60 docs,
// Query + Docs together 2000 characters, one request at a time.
"lkeap": {MaxDocuments: 60, MaxRequestChars: 2000, MaxConcurrency: 1},
// Ark Knowledge Service rerank: 50 per request.
"volcengine": {MaxDocuments: 50, MaxConcurrency: 4},
// NIM reranking: passages is capped at 512 items.
"nvidia": {MaxDocuments: 512},
// cloud.baidu.com/doc/qianfan-api: 文本数量不超过64, query 不超过
// 1600 个字符, 每条 document 不超过 4096 个字符.
"qianfan": {MaxDocuments: 64, MaxQueryChars: 1600, MaxDocumentChars: 4096},
// help.aliyun.com text-rerank: 500 documents per request. Its length
// limits are stated in tokens, which runes cannot express.
"aliyun": {MaxDocuments: 500},
} {
t.Run(id, func(t *testing.T) {
resolved, err := catalog.Resolve(catalog.Ref{
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
got := resolved.Rerank
assert.Equal(t, want.MaxDocuments, got.MaxDocuments, "max documents")
assert.Equal(t, want.MaxQueryChars, got.MaxQueryChars, "max query characters")
assert.Equal(t, want.MaxDocumentChars, got.MaxDocumentChars, "max document characters")
assert.Equal(t, want.MaxRequestChars, got.MaxRequestChars, "max request characters")
assert.Equal(t, want.MaxConcurrency, got.MaxConcurrency, "max concurrency")
})
}
}
// TestWeKnoraCloudRerankKeepsItsDeadline pins the one rerank vendor that has
// ever had a client-level timeout. Without it a hung endpoint holds the
// retrieval stage open for as long as the caller's context allows.
func TestWeKnoraCloudRerankKeepsItsDeadline(t *testing.T) {
resolved, err := catalog.Resolve(catalog.Ref{
Provider: "weknoracloud", Model: "rerank", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
assert.Equal(t, 60, resolved.Rerank.RequestTimeout)
// Everyone else has always run without one and relies on the caller.
for _, id := range []string{"zhipu", "jina", "siliconflow", "aliyun", "nvidia", "generic"} {
other, err := catalog.Resolve(catalog.Ref{
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
assert.Zero(t, other.Rerank.RequestTimeout, "%s did not have a client deadline before", id)
}
}
// TestTruncatePromptTokensIsReachableFromTheEditor closes the loop between the
// vendor accepting the extension and an operator being able to turn it on.
// Before the catalog it was readable from extra_config but had no input in the
// model editor, so the only way to set it was the API or the database.
// GET /models/providers renders extra fields dynamically, so declaring one is
// all it takes — and it must be declared exactly where the extension is
// accepted, or the form offers a switch the vendor will reject.
func TestTruncatePromptTokensIsReachableFromTheEditor(t *testing.T) {
for _, v := range catalog.ListByType(types.ModelTypeRerank) {
var field *catalog.ExtraField
for i := range v.ExtraFields {
if v.ExtraFields[i].Key == catalog.ExtraTruncatePromptTokens {
field = &v.ExtraFields[i]
}
}
resolved, err := catalog.Resolve(catalog.Ref{
Provider: v.ID, Model: "m", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
if !resolved.Rerank.AcceptsTruncatePromptTokens {
assert.Nil(t, field, "%s does not accept the extension, so it must not offer the input", v.ID)
continue
}
require.NotNil(t, field, "%s accepts the extension but offers no way to set it", v.ID)
assert.Equal(t, "number", field.Type)
assert.False(t, field.Required, "the extension is opt-in")
assert.Equal(t, []types.ModelType{types.ModelTypeRerank}, field.ModelTypes,
"the input belongs to the rerank form only")
}
}
// TestRerankScoreScalesMatchTheVendorDocs pins which vendors return something
// other than a 0..1 relevance score. Getting this wrong is silent: the number
// still looks like a score, and the retrieval threshold still compares it, so
// a logit-scaled vendor simply loses every negatively scored document.
func TestRerankScoreScalesMatchTheVendorDocs(t *testing.T) {
// Both spell the field like a probability and return neither.
// NIM: rankings[].logit, e.g. 0.226 / -1.171 / -6.31
// GPUStack: relevance_score, e.g. 1.951 / -3.734 / -6.157
logitScaled := map[string]bool{"nvidia": true, "gpustack": true}
for _, v := range catalog.ListByType(types.ModelTypeRerank) {
resolved, err := catalog.Resolve(catalog.Ref{
Provider: v.ID, Model: "m", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
want := api.ScoreProbability
if logitScaled[v.ID] {
want = api.ScoreLogit
}
assert.Equal(t, want, resolved.Rerank.ScoreScale, "%s score scale", v.ID)
}
}
// TestUnimplementedRerankDialectsAreRefused covers a model that names a
// protocol no package implements. Hiding it from the picker stops new rows;
// a row that already names it has to fail somewhere, and failing at
// construction with the reason beats sending a request shaped for the wrong
// protocol and reporting whatever the decoder makes of the reply.
func TestUnimplementedRerankDialectsAreRefused(t *testing.T) {
v, ok := catalog.Get("aliyun")
require.True(t, ok)
offered := make([]string, 0)
for _, m := range v.ModelsByType(types.ModelTypeRerank) {
offered = append(offered, m.ID)
}
assert.NotContains(t, offered, "qwen3-rerank", "the picker must not offer an unimplemented dialect")
assert.Contains(t, offered, "gte-rerank-v2")
_, err := catalog.Resolve(catalog.Ref{
Provider: "aliyun", Model: "qwen3-rerank", ModelType: types.ModelTypeRerank,
})
require.Error(t, err)
assert.Contains(t, err.Error(), "/compatible-api/v1/reranks",
"the error should name the protocol the model actually speaks")
// The vendor's other rerank model is unaffected.
_, err = catalog.Resolve(catalog.Ref{
Provider: "aliyun", Model: "gte-rerank-v2", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
}
// TestGatewayRerankEndpoints pins the URL each gateway's rerank rows reach.
// All three serve the Cohere dialect the protocol package already speaks, so
// opening the type was a declaration — but LiteLLM documents the route on the
// proxy root rather than under /v1, which is why it needs its own default.
func TestGatewayRerankEndpoints(t *testing.T) {
for _, tc := range []struct {
provider string
wantURL string
}{
// https://docs.novita.ai/api-reference/model-apis-llm-create-rerank
{provider: "novita", wantURL: "https://api.novita.ai/openai/v1/rerank"},
// https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request
{provider: "openrouter", wantURL: "https://openrouter.ai/api/v1/rerank"},
// https://docs.litellm.ai/docs/rerank — the curl example posts to
// http://0.0.0.0:4000/rerank, not /v1/rerank.
{provider: "litellm", wantURL: "http://your_litellm_proxy/rerank"},
} {
t.Run(tc.provider, func(t *testing.T) {
resolved, err := catalog.Resolve(catalog.Ref{
Provider: tc.provider, Model: "m", ModelType: types.ModelTypeRerank,
})
require.NoError(t, err)
require.Equal(t, api.RerankCohere, resolved.RerankAPI)
// Rebuild the URL the way cohererank does.
got := strings.TrimRight(resolved.BaseURL, "/")
if !strings.HasSuffix(got, "/rerank") {
got += "/rerank"
}
assert.Equal(t, tc.wantURL, got)
})
}
}
// TestScoreScaleIsOverridablePerRow covers the gateways, where the vendor
// cannot know the answer: GPUStack and a generic endpoint serve whatever
// reranker was deployed behind them, and the two families disagree — BGE
// answers a 0..1 probability, Qwen3-Reranker an unbounded score. Converting a
// probability as if it were a logit squeezes every score into [0.5, 0.73],
// which makes a relevance threshold meaningless, so the operator needs a way
// to say which one is actually there.
func TestScoreScaleIsOverridablePerRow(t *testing.T) {
for _, id := range []string{"gpustack", "generic"} {
t.Run(id, func(t *testing.T) {
base := "http://127.0.0.1:9/v1"
overridden, err := catalog.Resolve(catalog.Ref{
Provider: id, Model: "bge-reranker-v2-m3", ModelType: types.ModelTypeRerank,
BaseURL: base, Extra: map[string]string{catalog.ExtraScoreScale: "probability"},
})
require.NoError(t, err)
assert.Equal(t, api.ScoreProbability, overridden.Rerank.ScoreScale)
// And the input to set it is rendered by the editor.
v, ok := catalog.Get(id)
require.True(t, ok)
var field *catalog.ExtraField
for i := range v.ExtraFields {
if v.ExtraFields[i].Key == catalog.ExtraScoreScale {
field = &v.ExtraFields[i]
}
}
require.NotNil(t, field, "%s must offer the override it needs", id)
assert.Equal(t, "select", field.Type)
assert.Len(t, field.Options, 2)
})
}
_, err := catalog.Resolve(catalog.Ref{
Provider: "gpustack", Model: "m", ModelType: types.ModelTypeRerank,
BaseURL: "http://127.0.0.1:9/v1",
Extra: map[string]string{catalog.ExtraScoreScale: "sigmoid"},
})
require.Error(t, err, "an unknown scale must not be treated as a probability")
}
// TestGatewayVendorsOfferTheScoreScaleOverride keeps the override where the
// question is open. A gateway cannot know which reranker is deployed behind
// it, so all three offer the input; the other vendors' scales are settled by
// their own documentation and must not offer a switch that only lets an
// operator get it wrong.
func TestGatewayVendorsOfferTheScoreScaleOverride(t *testing.T) {
gateways := map[string]bool{"generic": true, "gpustack": true, "litellm": true}
for _, v := range catalog.ListByType(types.ModelTypeRerank) {
var field *catalog.ExtraField
for i := range v.ExtraFields {
if v.ExtraFields[i].Key == catalog.ExtraScoreScale {
field = &v.ExtraFields[i]
}
}
if !gateways[v.ID] {
assert.Nil(t, field, "%s: its documentation settles the scale", v.ID)
continue
}
require.NotNil(t, field, "%s is a gateway and needs the override", v.ID)
require.Len(t, field.Options, 2)
for _, opt := range field.Options {
assert.NotEmpty(t, opt.LocalizedLabel("zh-CN"), "%s: option %q has no label", v.ID, opt.Value)
assert.NotEqual(t, opt.Label, opt.LocalizedLabel("zh-CN"),
"%s: option %q is prose and needs a zh-CN label", v.ID, opt.Value)
}
assert.NotEqual(t, field.Placeholder, field.LocalizedPlaceholder("zh-CN"),
"%s: the placeholder is prose and needs a zh-CN variant", v.ID)
}
}
// Every operator-facing string a vendor declares has to exist in both
// languages, or half the product reads English prose in a Chinese form.
func TestEveryExtraFieldProseIsLocalized(t *testing.T) {
for _, v := range catalog.List() {
for _, field := range v.ExtraFields {
if field.Placeholder != "" && strings.Contains(field.Placeholder, " ") {
assert.NotEqual(t, field.Placeholder, field.LocalizedPlaceholder("zh-CN"),
"%s/%s: placeholder is prose without a zh-CN variant", v.ID, field.Key)
}
for _, opt := range field.Options {
// An identifier (a region code, a version) is the same in
// every language; prose is not.
if strings.Contains(opt.Label, " ") {
assert.NotEqual(t, opt.Label, opt.LocalizedLabel("zh-CN"),
"%s/%s: option %q is prose without a zh-CN label", v.ID, field.Key, opt.Value)
}
}
}
}
}

View File

@ -1,172 +0,0 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// AliyunReranker implements a reranking system based on Aliyun DashScope models
type AliyunReranker struct {
modelName string // Name of the model used for reranking
modelID string // Unique identifier of the model
apiKey string // API key for authentication
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
customHeaders map[string]string
}
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
func (r *AliyunReranker) SetCustomHeaders(headers map[string]string) {
r.customHeaders = headers
}
// AliyunRerankRequest represents a request to rerank documents using Aliyun DashScope API
type AliyunRerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Input AliyunRerankInput `json:"input"` // Input containing query and documents
Parameters AliyunRerankParameters `json:"parameters"` // Parameters for the reranking
}
// AliyunRerankInput contains the query and documents for reranking
type AliyunRerankInput struct {
Query string `json:"query"` // Query text to compare documents against
Documents []string `json:"documents"` // List of document texts to rerank
}
// AliyunRerankParameters contains parameters for the reranking request
type AliyunRerankParameters struct {
ReturnDocuments bool `json:"return_documents"` // Whether to return documents in response
TopN int `json:"top_n"` // Number of top results to return
}
// AliyunRerankResponse represents the response from Aliyun DashScope reranking request
type AliyunRerankResponse struct {
Output AliyunOutput `json:"output"` // Output containing results
Usage AliyunUsage `json:"usage"` // Token usage information
}
// AliyunOutput contains the reranking results
type AliyunOutput struct {
Results []AliyunRankResult `json:"results"` // Ranked results with relevance scores
}
// AliyunRankResult represents a single reranking result from Aliyun
type AliyunRankResult struct {
Document AliyunDocument `json:"document"` // Document information
Index int `json:"index"` // Original index of the document
RelevanceScore float64 `json:"relevance_score"` // Relevance score
}
// AliyunDocument represents document information in Aliyun response
type AliyunDocument struct {
Text string `json:"text"` // Document text
}
// AliyunUsage contains information about token usage in the Aliyun API request
type AliyunUsage struct {
TotalTokens int `json:"total_tokens"` // Total tokens consumed
}
// NewAliyunReranker creates a new instance of Aliyun reranker with the provided configuration
func NewAliyunReranker(config *RerankerConfig) (*AliyunReranker, error) {
apiKey := config.APIKey
baseURL := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
if url := config.BaseURL; url != "" {
baseURL = url
}
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
return &AliyunReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
}, nil
}
// Rerank performs document reranking based on relevance to the query using Aliyun DashScope API
func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body
requestBody := &AliyunRerankRequest{
Model: r.modelName,
Input: AliyunRerankInput{
Query: query,
Documents: documents,
},
Parameters: AliyunRerankParameters{
ReturnDocuments: true,
TopN: len(documents), // Return all documents
},
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
// Send the request
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
secutils.ApplyCustomHeaders(req, r.customHeaders)
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("do request: %w", err)
}
defer resp.Body.Close()
// Read the response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("aliyun rerank API error: Http Status: %s, Body: %s", resp.Status, string(body))
}
var response AliyunRerankResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
// Convert Aliyun results to standard RankResult format
results := make([]RankResult, len(response.Output.Results))
for i, aliyunResult := range response.Output.Results {
results[i] = RankResult{
Index: aliyunResult.Index,
Document: DocumentInfo{
Text: aliyunResult.Document.Text,
},
RelevanceScore: aliyunResult.RelevanceScore,
}
}
return results, nil
}
// GetModelName returns the name of the reranking model
func (r *AliyunReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the unique identifier of the reranking model
func (r *AliyunReranker) GetModelID() string {
return r.modelID
}

View File

@ -1,128 +0,0 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// JinaReranker implements a reranking system using Jina AI API
// Jina API uses different parameters than standard OpenAI-compatible APIs
type JinaReranker struct {
modelName string // Name of the model used for reranking
modelID string // Unique identifier of the model
apiKey string // API key for authentication
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
customHeaders map[string]string
}
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
func (r *JinaReranker) SetCustomHeaders(headers map[string]string) {
r.customHeaders = headers
}
// JinaRerankRequest represents a Jina rerank request
// Note: Jina does NOT support truncate_prompt_tokens parameter
type JinaRerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Query string `json:"query"` // Query text to compare documents against
Documents []string `json:"documents"` // List of document texts to rerank
TopN int `json:"top_n,omitempty"` // Number of top results to return
ReturnDocuments bool `json:"return_documents,omitempty"` // Whether to return document text in response
}
// JinaRerankResponse represents the response from a Jina reranking request
type JinaRerankResponse struct {
Model string `json:"model"` // Model used for reranking
Results []RankResult `json:"results"` // Ranked results with relevance scores
Usage struct {
TotalTokens int `json:"total_tokens"` // Total tokens consumed
} `json:"usage"`
}
// NewJinaReranker creates a new instance of Jina reranker with the provided configuration
func NewJinaReranker(config *RerankerConfig) (*JinaReranker, error) {
apiKey := config.APIKey
baseURL := "https://api.jina.ai/v1"
if url := config.BaseURL; url != "" {
baseURL = url
}
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
return &JinaReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
}, nil
}
// Rerank performs document reranking based on relevance to the query
func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body - Jina does NOT use truncate_prompt_tokens
requestBody := &JinaRerankRequest{
Model: r.modelName,
Query: query,
Documents: documents,
ReturnDocuments: true, // Return document text in response
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
// Send the request
req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
secutils.ApplyCustomHeaders(req, r.customHeaders)
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("do request: %w", err)
}
defer resp.Body.Close()
// Read the response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body))
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
}
var response JinaRerankResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
return response.Results, nil
}
// GetModelName returns the name of the reranking model
func (r *JinaReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the unique identifier of the reranking model
func (r *JinaReranker) GetModelID() string {
return r.modelID
}

View File

@ -4,19 +4,15 @@ import (
"context"
"fmt"
"strings"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
lkeap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lkeap/v20240522"
)
const (
// LKEAPMaxDocumentsPerRequest is the maximum number of documents accepted by RunRerank.
LKEAPMaxDocumentsPerRequest = 60
// LKEAPMaxRequestCharacters is the maximum combined length of Query and Docs accepted by RunRerank.
LKEAPMaxRequestCharacters = 2000
// LKEAPRerankEndpoint 腾讯云知识引擎原子能力 Rerank API 域名
LKEAPRerankEndpoint = "lkeap.tencentcloudapi.com"
// LKEAPDefaultRegion RunRerank 支持的地域,默认广州
@ -25,16 +21,17 @@ const (
LKEAPDefaultRerankModel = "lke-reranker-base"
)
// LKEAPReranker 使用腾讯云知识引擎原子能力 RunRerank 接口进行重排序。
// 鉴权使用腾讯云 API 密钥:APIKey 为 SecretId,AppSecret 为 SecretKey。
type LKEAPReranker struct {
// lkeapClient calls Tencent Cloud's RunRerank action through the official
// SDK, which owns the TC3 signing this API requires. It implements
// api.Reranker and nothing else: the per-request ceilings (60 documents,
// 2000 characters for Query plus Docs together) are declared on the vendor
// and enforced by protocolReranker, like every other vendor's.
type lkeapClient struct {
modelName string
modelID string
client *lkeap.Client
}
// NewLKEAPReranker 创建 LKEAP rerank 客户端。
func NewLKEAPReranker(config *RerankerConfig) (*LKEAPReranker, error) {
func newLKEAPClient(config *RerankerConfig, resolved *catalog.Resolved) (api.Reranker, error) {
secretID := strings.TrimSpace(config.APIKey)
secretKey := strings.TrimSpace(config.AppSecret)
if secretKey == "" && config.ExtraConfig != nil {
@ -60,86 +57,23 @@ func NewLKEAPReranker(config *RerankerConfig) (*LKEAPReranker, error) {
return nil, fmt.Errorf("create LKEAP client: %w", err)
}
modelName := strings.TrimSpace(config.ModelName)
modelName := strings.TrimSpace(resolved.RemoteModel)
if modelName == "" {
modelName = LKEAPDefaultRerankModel
}
return &LKEAPReranker{
modelName: modelName,
modelID: config.ModelID,
client: client,
}, nil
return &lkeapClient{modelName: modelName, client: client}, nil
}
// Rerank 调用 RunRerank 对文档按与 query 的相关性打分。
func (r *LKEAPReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
if len(documents) == 0 {
return []RankResult{}, nil
}
batches, err := lkeapRerankBatches(query, documents)
if err != nil {
return nil, err
}
results := make([]RankResult, 0, len(documents))
for _, batch := range batches {
batchResults, err := r.rerankBatch(ctx, query, batch.documents)
if err != nil {
return nil, err
}
for i := range batchResults {
batchResults[i].Index += batch.start
}
results = append(results, batchResults...)
}
return results, nil
}
type lkeapRerankBatch struct {
start int
documents []string
}
func lkeapRerankBatches(query string, documents []string) ([]lkeapRerankBatch, error) {
queryLength := utf8.RuneCountInString(query)
if queryLength >= LKEAPMaxRequestCharacters {
return nil, fmt.Errorf("LKEAP rerank query is %d characters; Query and Docs together support at most %d characters", queryLength, LKEAPMaxRequestCharacters)
}
batches := make([]lkeapRerankBatch, 0, (len(documents)+LKEAPMaxDocumentsPerRequest-1)/LKEAPMaxDocumentsPerRequest)
batchStart := 0
batchLength := queryLength
batchDocuments := make([]string, 0, LKEAPMaxDocumentsPerRequest)
for index, document := range documents {
documentLength := utf8.RuneCountInString(document)
if queryLength+documentLength > LKEAPMaxRequestCharacters {
return nil, fmt.Errorf("LKEAP rerank document at index %d is %d characters; Query and each document together support at most %d characters", index, documentLength, LKEAPMaxRequestCharacters)
}
if len(batchDocuments) == LKEAPMaxDocumentsPerRequest || batchLength+documentLength > LKEAPMaxRequestCharacters {
batches = append(batches, lkeapRerankBatch{start: batchStart, documents: batchDocuments})
batchStart = index
batchLength = queryLength
batchDocuments = make([]string, 0, LKEAPMaxDocumentsPerRequest)
}
batchDocuments = append(batchDocuments, document)
batchLength += documentLength
}
if len(batchDocuments) > 0 {
batches = append(batches, lkeapRerankBatch{start: batchStart, documents: batchDocuments})
}
return batches, nil
}
func (r *LKEAPReranker) rerankBatch(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Rerank scores one batch. RunRerank answers a bare score list in input
// order rather than an indexed, sorted result set.
func (r *lkeapClient) Rerank(
ctx context.Context, query string, documents []string,
) ([]api.RerankResult, error) {
req := lkeap.NewRunRerankRequest()
req.Query = common.StringPtr(query)
req.Docs = common.StringPtrs(documents)
req.Model = common.StringPtr(r.modelName)
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, LKEAPRerankEndpoint, query, documents))
resp, err := r.client.RunRerankWithContext(ctx, req)
if err != nil {
return nil, fmt.Errorf("LKEAP RunRerank: %w", err)
@ -147,35 +81,18 @@ func (r *LKEAPReranker) rerankBatch(ctx context.Context, query string, documents
if resp == nil || resp.Response == nil || len(resp.Response.ScoreList) == 0 {
return nil, fmt.Errorf("LKEAP rerank API returned empty score list")
}
scores := resp.Response.ScoreList
if len(scores) != len(documents) {
return nil, fmt.Errorf("LKEAP rerank score count mismatch: got %d scores for %d documents",
len(scores), len(documents))
}
results := make([]RankResult, len(documents))
results := make([]api.RerankResult, len(documents))
for i, score := range scores {
if score == nil {
continue
}
results[i] = RankResult{
Index: i,
Document: DocumentInfo{
Text: documents[i],
},
RelevanceScore: *score,
results[i] = api.RerankResult{Index: i, Text: documents[i]}
if score != nil {
results[i].Score = *score
}
}
return results, nil
}
// GetModelName returns the rerank model name.
func (r *LKEAPReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the model ID.
func (r *LKEAPReranker) GetModelID() string {
return r.modelID
}

View File

@ -1,178 +0,0 @@
package rerank
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
lkeap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lkeap/v20240522"
)
func TestNewLKEAPReranker_requiresCredentials(t *testing.T) {
_, err := NewLKEAPReranker(&RerankerConfig{
ModelName: "lke-reranker-base",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "secret_id")
}
func TestNewLKEAPReranker_secretKeyFromExtraConfig(t *testing.T) {
r, err := NewLKEAPReranker(&RerankerConfig{
APIKey: "AKIDtest",
ModelName: "lke-reranker-base",
ExtraConfig: map[string]string{
"secret_key": "sk-test",
"region": "ap-beijing",
},
})
require.NoError(t, err)
require.NotNil(t, r)
assert.Equal(t, "lke-reranker-base", r.GetModelName())
}
func TestNewLKEAPReranker_defaultModelName(t *testing.T) {
r, err := NewLKEAPReranker(&RerankerConfig{
APIKey: "AKIDtest",
AppSecret: "sk-test",
})
require.NoError(t, err)
assert.Equal(t, LKEAPDefaultRerankModel, r.GetModelName())
}
func TestLKEAPReranker_Rerank_emptyDocuments(t *testing.T) {
r, err := NewLKEAPReranker(&RerankerConfig{
APIKey: "AKIDtest",
AppSecret: "sk-test",
})
require.NoError(t, err)
results, err := r.Rerank(t.Context(), "query", nil)
require.NoError(t, err)
assert.Empty(t, results)
}
func TestLKEAPReranker_Rerank_batchesMoreThan60Documents(t *testing.T) {
var batches [][]string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
var request struct {
Docs []string `json:"Docs"`
}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
batches = append(batches, request.Docs)
scores := make([]float64, len(request.Docs))
for i, doc := range request.Docs {
index, err := lkeapTestDocumentIndex(doc)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
scores[i] = float64(index)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"Response": map[string]any{
"ScoreList": scores,
"RequestId": "test-request",
},
})
}))
defer server.Close()
r := newTestLKEAPReranker(t, server.URL)
documents := make([]string, 61)
for i := range documents {
documents[i] = strconv.Itoa(i) + ":document"
}
results, err := r.Rerank(t.Context(), "query", documents)
require.NoError(t, err)
require.Len(t, results, len(documents))
require.Len(t, batches, 2)
assert.Len(t, batches[0], 60)
assert.Len(t, batches[1], 1)
for i, result := range results {
assert.Equal(t, i, result.Index)
assert.Equal(t, documents[i], result.Document.Text)
assert.Equal(t, float64(i), result.RelevanceScore)
}
}
func TestLKEAPReranker_Rerank_batchesWithinCharacterLimit(t *testing.T) {
var batches [][]string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
var request struct {
Docs []string `json:"Docs"`
}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
batches = append(batches, request.Docs)
scores := make([]float64, len(request.Docs))
for i, doc := range request.Docs {
index, err := lkeapTestDocumentIndex(doc)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
scores[i] = float64(index)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"Response": map[string]any{
"ScoreList": scores,
"RequestId": "test-request",
},
})
}))
defer server.Close()
r := newTestLKEAPReranker(t, server.URL)
documents := []string{
"0:" + strings.Repeat("a", 1000),
"1:" + strings.Repeat("b", 1000),
"2:" + strings.Repeat("c", 1000),
}
results, err := r.Rerank(t.Context(), "query", documents)
require.NoError(t, err)
require.Len(t, results, len(documents))
require.Len(t, batches, 3)
for i, batch := range batches {
assert.Len(t, batch, 1)
assert.Equal(t, documents[i], batch[0])
}
}
func newTestLKEAPReranker(t *testing.T, serverURL string) *LKEAPReranker {
t.Helper()
endpoint, err := url.Parse(serverURL)
require.NoError(t, err)
clientProfile := profile.NewClientProfile()
clientProfile.HttpProfile.Endpoint = endpoint.Host
clientProfile.HttpProfile.Scheme = "HTTP"
client, err := lkeap.NewClient(common.NewCredential("AKIDtest", "sk-test"), LKEAPDefaultRegion, clientProfile)
require.NoError(t, err)
return &LKEAPReranker{
modelName: LKEAPDefaultRerankModel,
client: client,
}
}
func lkeapTestDocumentIndex(document string) (int, error) {
value, _, _ := strings.Cut(document, ":")
return strconv.Atoi(value)
}

View File

@ -1,156 +0,0 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// NvidiaReranker implements a reranking system using Jina AI API
// Jina API uses different parameters than standard OpenAI-compatible APIs
type NvidiaReranker struct {
modelName string // Name of the model used for reranking
modelID string // Unique identifier of the model
apiKey string // API key for authentication
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
customHeaders map[string]string
}
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
func (r *NvidiaReranker) SetCustomHeaders(headers map[string]string) {
r.customHeaders = headers
}
type NvidiaRerankDocument struct {
Text string `json:"text"`
}
// NvidiaRerankRequest represents a Jina rerank request
// Note: Jina does NOT support truncate_prompt_tokens parameter
type NvidiaRerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Query NvidiaRerankDocument `json:"query"` // Query text to compare documents against
Documents []NvidiaRerankDocument `json:"passages"` // List of document texts to rerank
}
type NvidiaRankResult struct {
Index int `json:"index"`
Logit float64 `json:"logit"`
}
// NvidiaRerankResponse represents the response from a Jina reranking request
type NvidiaRerankResponse struct {
Model string `json:"model"` // Model used for reranking
Results []NvidiaRankResult `json:"rankings"` // Ranked results with relevance scores
}
// NewNvidiaReranker creates a new instance of Jina reranker with the provided configuration
func NewNvidiaReranker(config *RerankerConfig) (*NvidiaReranker, error) {
apiKey := config.APIKey
baseURL := "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking"
if url := config.BaseURL; url != "" {
baseURL = url
}
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
return &NvidiaReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
}, nil
}
// Rerank performs document reranking based on relevance to the query
func (r *NvidiaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body - Jina does NOT use truncate_prompt_tokens
requestBody := &NvidiaRerankRequest{
Model: r.modelName,
Query: NvidiaRerankDocument{Text: query},
Documents: make([]NvidiaRerankDocument, len(documents)),
}
for i := range requestBody.Documents {
requestBody.Documents[i].Text = documents[i]
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
// Send the request
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
secutils.ApplyCustomHeaders(req, r.customHeaders)
// Log the curl equivalent for debugging (API key masked for security)
logger.GetLogger(ctx).Infof(
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
r.baseURL, string(jsonData),
)
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("do request: %w", err)
}
defer resp.Body.Close()
// Read the response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body))
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
}
var response NvidiaRerankResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
ret := make([]RankResult, len(response.Results))
for i, result := range response.Results {
ret[i] = RankResult{
Index: result.Index,
Document: DocumentInfo{Text: documents[result.Index]},
RelevanceScore: normalizeNvidiaLogit(result.Logit),
}
}
return ret, nil
}
// normalizeNvidiaLogit converts NVIDIA's raw reranker logit into a probability.
func normalizeNvidiaLogit(logit float64) float64 {
if logit >= 0 {
return 1 / (1 + math.Exp(-logit))
}
expLogit := math.Exp(logit)
return expLogit / (1 + expLogit)
}
// GetModelName returns the name of the reranking model
func (r *NvidiaReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the unique identifier of the reranking model
func (r *NvidiaReranker) GetModelID() string {
return r.modelID
}

View File

@ -1,49 +0,0 @@
package rerank
import (
"math"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNvidiaRerankerNormalizesLogitsToProbabilities(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodPost, r.Method)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"model": "nvidia-rerank-model",
"rankings": [
{"index": 0, "logit": 23.0},
{"index": 1, "logit": 0.0},
{"index": 2, "logit": -23.0}
]
}`))
}))
defer server.Close()
reranker := &NvidiaReranker{
modelName: "nvidia-rerank-model",
apiKey: "nvapi-test",
baseURL: server.URL,
client: server.Client(),
}
results, err := reranker.Rerank(t.Context(), "query", []string{"high", "neutral", "low"})
require.NoError(t, err)
require.Len(t, results, 3)
logits := []float64{23, 0, -23}
for i, logit := range logits {
want := 1 / (1 + math.Exp(-logit))
assert.InDelta(t, want, results[i].RelevanceScore, 1e-12)
assert.Greater(t, results[i].RelevanceScore, 0.0)
assert.Less(t, results[i].RelevanceScore, 1.0)
}
assert.Greater(t, results[0].RelevanceScore, results[1].RelevanceScore)
assert.Greater(t, results[1].RelevanceScore, results[2].RelevanceScore)
}

View File

@ -0,0 +1,142 @@
package rerank
import (
"context"
"fmt"
"math"
"sort"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"golang.org/x/sync/errgroup"
)
// protocolReranker adapts a protocol client to the Reranker interface and
// owns the two things every vendor needs and none of them should implement
// itself: splitting a candidate set that exceeds the documented per-request
// ceilings, and putting the returned scores on one scale.
type protocolReranker struct {
inner api.Reranker
settings catalog.RerankSettings
endpoint string
modelName string
modelID string
}
// defaultBatchConcurrency bounds in-flight batches when a vendor declares no
// preference, so a large embedding_top_k cannot fan out into an unbounded
// burst of requests.
const defaultBatchConcurrency = 4
func (r *protocolReranker) GetModelName() string { return r.modelName }
func (r *protocolReranker) GetModelID() string { return r.modelID }
func (r *protocolReranker) Rerank(
ctx context.Context, query string, documents []string,
) ([]RankResult, error) {
if len(documents) == 0 {
return nil, nil
}
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.endpoint, query, documents))
// The query travels in every request, so an over-long one cannot be made
// to fit by splitting the documents. SplitBatches charges it to each
// batch but only against a whole-request budget; a vendor that caps the
// query on its own is checked here.
if limit := r.settings.MaxQueryChars; limit > 0 {
if length := utf8.RuneCountInString(query); length > limit {
return nil, fmt.Errorf(
"%s rerank: query is %d characters; the limit is %d",
r.modelName, length, limit,
)
}
}
batches, err := api.SplitBatches(documents, utf8.RuneCountInString(query), r.settings.BatchLimits())
if err != nil {
return nil, fmt.Errorf("%s rerank: %w", r.modelName, err)
}
// Each batch is scored against the same query independently, so the
// per-batch results are comparable and are merged and re-ranked below.
scored := make([][]api.RerankResult, len(batches))
group, groupCtx := errgroup.WithContext(ctx)
group.SetLimit(r.concurrency())
for i, batch := range batches {
group.Go(func() error {
out, err := r.inner.Rerank(groupCtx, query, batch.Items)
if err != nil {
return err
}
for j := range out {
out[j].Index += batch.Start
}
scored[i] = out
return nil
})
}
if err := group.Wait(); err != nil {
return nil, err
}
results := make([]RankResult, 0, len(documents))
for _, batch := range scored {
for _, item := range batch {
if item.Index < 0 || item.Index >= len(documents) {
return nil, fmt.Errorf(
"%s rerank: index %d out of range for %d documents",
r.modelName, item.Index, len(documents),
)
}
text := item.Text
if text == "" {
text = documents[item.Index]
}
results = append(results, RankResult{
Index: item.Index,
Document: DocumentInfo{Text: text},
RelevanceScore: normalizeScore(item.Score, r.settings.ScoreScale),
})
}
}
// A single request comes back ranked, and the retrieval pipeline reads
// the slice as ranked: it takes results[0] as the best candidate for its
// threshold fallback. Concatenating per-batch results would make that the
// best of the first batch only, so the merged set is ordered once here.
// Ties keep the lower index, which keeps the order stable.
sort.SliceStable(results, func(i, j int) bool {
return results[i].RelevanceScore > results[j].RelevanceScore
})
return results, nil
}
func (r *protocolReranker) concurrency() int {
if r.settings.MaxConcurrency > 0 {
return r.settings.MaxConcurrency
}
return defaultBatchConcurrency
}
// normalizeScore puts every vendor's score on the 0..1 scale the retrieval
// pipeline compares against RerankThreshold.
//
// Most protocols already return a relevance probability. NIM returns the raw
// logit of its relevance head instead — unbounded and routinely negative —
// so a threshold tuned for probabilities would reject almost everything on
// that vendor. The logistic function is the inverse of a log-odds, so this is
// a unit conversion rather than a heuristic: it maps the vendor's own ranking
// onto the scale the rest of the system already speaks, order preserved.
func normalizeScore(score float64, scale api.ScoreScale) float64 {
if scale != api.ScoreLogit {
return score
}
// Two branches so neither exponential overflows: this is the form the
// pre-catalog NVIDIA client used, kept verbatim.
if score >= 0 {
return 1 / (1 + math.Exp(-score))
}
exp := math.Exp(score)
return exp / (1 + exp)
}

View File

@ -0,0 +1,187 @@
package rerank
import (
"context"
"fmt"
"math"
"strings"
"sync"
"testing"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeProtocol records every batch it was handed and scores documents by
// their position within the batch, so a wrong offset shows up immediately.
type fakeProtocol struct {
mu sync.Mutex
batches [][]string
err error
}
func (f *fakeProtocol) Rerank(
_ context.Context, _ string, documents []string,
) ([]api.RerankResult, error) {
f.mu.Lock()
f.batches = append(f.batches, append([]string(nil), documents...))
f.mu.Unlock()
if f.err != nil {
return nil, f.err
}
out := make([]api.RerankResult, len(documents))
for i, doc := range documents {
out[i] = api.RerankResult{Index: i, Score: float64(i), Text: doc}
}
return out, nil
}
func newWrapped(inner api.Reranker, settings catalog.RerankSettings) *protocolReranker {
return &protocolReranker{
inner: inner, settings: settings,
endpoint: "https://example.invalid", modelName: "m", modelID: "id",
}
}
// The documented per-request ceilings live on the vendor, and one shared
// layer enforces them. Indices must come back pointing into the caller's
// original slice, not into the batch.
func TestBatchingSplitsAndRestoresGlobalIndices(t *testing.T) {
fake := &fakeProtocol{}
r := newWrapped(fake, catalog.RerankSettings{MaxDocuments: 2, MaxConcurrency: 1})
docs := []string{"d0", "d1", "d2", "d3", "d4"}
got, err := r.Rerank(context.Background(), "q", docs)
require.NoError(t, err)
require.Len(t, fake.batches, 3, "5 documents at 2 per request is 3 requests")
indices := make([]int, 0, len(got))
for _, item := range got {
indices = append(indices, item.Index)
assert.Equal(t, docs[item.Index], item.Document.Text, "index must address the caller's slice")
}
assert.ElementsMatch(t, []int{0, 1, 2, 3, 4}, indices)
}
func TestBatchingIsSkippedWhenTheVendorDeclaresNoLimit(t *testing.T) {
fake := &fakeProtocol{}
r := newWrapped(fake, catalog.RerankSettings{})
_, err := r.Rerank(context.Background(), "q", []string{"a", "b", "c", "d"})
require.NoError(t, err)
assert.Len(t, fake.batches, 1)
}
// A document over the vendor's per-document ceiling cannot be scored. Saying
// so beats sending a prefix and returning a number for text nobody asked
// about.
func TestAnOversizedDocumentIsAnError(t *testing.T) {
r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxDocumentChars: 10})
_, err := r.Rerank(context.Background(), "q", []string{"ok", strings.Repeat("x", 99)})
require.Error(t, err)
assert.Contains(t, err.Error(), "index 1")
}
func TestBatchErrorsPropagate(t *testing.T) {
r := newWrapped(&fakeProtocol{err: fmt.Errorf("upstream exploded")}, catalog.RerankSettings{})
_, err := r.Rerank(context.Background(), "q", []string{"a"})
require.Error(t, err)
assert.Contains(t, err.Error(), "upstream exploded")
}
func TestEmptyInputMakesNoRequest(t *testing.T) {
fake := &fakeProtocol{}
r := newWrapped(fake, catalog.RerankSettings{})
got, err := r.Rerank(context.Background(), "q", nil)
require.NoError(t, err)
assert.Empty(t, got)
assert.Empty(t, fake.batches)
}
// NIM returns the raw logit of its relevance head. The retrieval pipeline
// compares scores against RerankThreshold, which is tuned for probabilities,
// so the logit is converted rather than passed through — otherwise the
// default threshold of 0.2 keeps only documents whose logit is positive.
func TestLogitScoresBecomeProbabilities(t *testing.T) {
for _, tc := range []struct {
logit float64
want float64
}{
{logit: 0.226318359375, want: 0.5563},
{logit: -1.171875, want: 0.2366},
{logit: -6.3125, want: 0.0018},
{logit: 0, want: 0.5},
} {
assert.InDelta(t, tc.want, normalizeScore(tc.logit, api.ScoreLogit), 1e-4,
"logit %v", tc.logit)
}
}
// The conversion is monotonic, so it re-scales without reordering.
func TestLogitConversionPreservesOrder(t *testing.T) {
logits := []float64{-6.3125, -1.171875, 0.226318359375, 4}
previous := -1.0
for _, logit := range logits {
got := normalizeScore(logit, api.ScoreLogit)
assert.Greater(t, got, previous)
previous = got
}
}
func TestProbabilityScoresPassThroughUntouched(t *testing.T) {
assert.Equal(t, 0.9819, normalizeScore(0.9819, api.ScoreProbability))
assert.Equal(t, 0.9819, normalizeScore(0.9819, ""))
}
// A vendor that caps the query cannot be satisfied by splitting documents,
// so the query is checked on its own before any request goes out.
func TestAnOversizedQueryIsAnError(t *testing.T) {
fake := &fakeProtocol{}
r := newWrapped(fake, catalog.RerankSettings{MaxQueryChars: 10})
_, err := r.Rerank(context.Background(), strings.Repeat("q", 11), []string{"d"})
require.Error(t, err)
assert.Contains(t, err.Error(), "query is 11 characters")
assert.Empty(t, fake.batches, "nothing should reach the vendor")
}
func TestAQueryWithinTheLimitPasses(t *testing.T) {
r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxQueryChars: 10})
_, err := r.Rerank(context.Background(), strings.Repeat("中", 10), []string{"d"})
require.NoError(t, err, "runes, not bytes")
}
// The retrieval pipeline reads results[0] as the best candidate. Batches come
// back each individually ranked, so the merged set has to be re-ranked or the
// caller would see the best of the first batch only.
func TestMergedBatchesAreRankedGlobally(t *testing.T) {
// Scores are the position within the batch, so batch 2 holds the highest.
r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxDocuments: 2, MaxConcurrency: 1})
got, err := r.Rerank(context.Background(), "q", []string{"d0", "d1", "d2", "d3", "d4"})
require.NoError(t, err)
require.Len(t, got, 5)
for i := 1; i < len(got); i++ {
assert.GreaterOrEqual(t, got[i-1].RelevanceScore, got[i].RelevanceScore,
"results must be ranked across batches, not concatenated")
}
}
// The pre-catalog NVIDIA client used a two-branch logistic so neither
// exponential overflows. These are the values its own test pinned.
func TestLogitConversionHandlesExtremes(t *testing.T) {
for _, logit := range []float64{23, 0, -23} {
got := normalizeScore(logit, api.ScoreLogit)
assert.InDelta(t, 1/(1+math.Exp(-logit)), got, 1e-12, "logit %v", logit)
assert.Greater(t, got, 0.0)
assert.Less(t, got, 1.0)
}
assert.False(t, math.IsNaN(normalizeScore(-1000, api.ScoreLogit)))
assert.False(t, math.IsNaN(normalizeScore(1000, api.ScoreLogit)))
}

View File

@ -1,153 +0,0 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// OpenAIReranker implements a reranking system based on OpenAI models
type OpenAIReranker struct {
modelName string // Name of the model used for reranking
modelID string // Unique identifier of the model
apiKey string // API key for authentication
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
customHeaders map[string]string
// truncatePromptTokens, when > 0, is sent as the vLLM-specific
// truncate_prompt_tokens request field. It must never be sent by default:
// providers that honor it (e.g. SiliconFlow) keep only the LAST N tokens of
// the templated rerank prompt, which cuts the query off long documents and
// collapses every relevance score to near zero (issue #2143).
truncatePromptTokens int
}
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
func (r *OpenAIReranker) SetCustomHeaders(headers map[string]string) {
r.customHeaders = headers
}
// RerankRequest represents a request to rerank documents based on relevance to a query
type RerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Query string `json:"query"` // Query text to compare documents against
Documents []string `json:"documents"` // List of document texts to rerank
AdditionalData map[string]interface{} `json:"additional_data,omitempty"` // Optional additional data for the model
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` // Maximum prompt tokens to use (vLLM-specific, opt-in)
}
// RerankResponse represents the response from a reranking request
type RerankResponse struct {
ID string `json:"id"` // Request ID
Model string `json:"model"` // Model used for reranking
Usage UsageInfo `json:"usage"` // Token usage information
Results []RankResult `json:"results"` // Ranked results with relevance scores
}
// UsageInfo contains information about token usage in the API request
type UsageInfo struct {
TotalTokens int `json:"total_tokens"` // Total tokens consumed
}
// NewOpenAIReranker creates a new instance of OpenAI reranker with the provided configuration
func NewOpenAIReranker(config *RerankerConfig) (*OpenAIReranker, error) {
apiKey := config.APIKey
baseURL := "https://api.openai.com/v1"
if url := config.BaseURL; url != "" {
baseURL = url
}
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
// Optional opt-in for vLLM-style deployments that need server-side prompt
// truncation. Configured via extra_config; never enabled by default.
truncatePromptTokens := 0
if config.ExtraConfig != nil {
if raw := strings.TrimSpace(config.ExtraConfig["truncate_prompt_tokens"]); raw != "" {
n, err := strconv.Atoi(raw)
if err != nil || n <= 0 {
return nil, fmt.Errorf("invalid truncate_prompt_tokens in extra_config: %q", raw)
}
truncatePromptTokens = n
}
}
return &OpenAIReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
truncatePromptTokens: truncatePromptTokens,
}, nil
}
// Rerank performs document reranking based on relevance to the query
func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body. truncate_prompt_tokens is only included when
// explicitly configured: sending it unconditionally corrupts scores on
// providers that honor it (see OpenAIReranker.truncatePromptTokens).
requestBody := &RerankRequest{
Model: r.modelName,
Query: query,
Documents: documents,
TruncatePromptTokens: r.truncatePromptTokens,
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
// Send the request
req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
secutils.ApplyCustomHeaders(req, r.customHeaders)
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("do request: %w", err)
}
defer resp.Body.Close()
// Read the response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
}
var response RerankResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
return response.Results, nil
}
// GetModelName returns the name of the reranking model
func (r *OpenAIReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the unique identifier of the reranking model
func (r *OpenAIReranker) GetModelID() string {
return r.modelID
}

View File

@ -1,157 +0,0 @@
package rerank
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
// newRerankScoreTestServer emulates an OpenAI-compatible /rerank endpoint the
// way a vLLM-backed provider (e.g. SiliconFlow) behaves, and records the last
// decoded request body.
//
// If the request carries truncate_prompt_tokens, the backend keeps only the
// last N tokens of the templated rerank prompt — the query gets cut off long
// documents and every relevance score collapses to near zero (issue #2143).
// Otherwise it returns the real scores, sorted by relevance_score descending,
// with index pointing back at the original input-document position.
func newRerankScoreTestServer(t *testing.T, lastRequest *map[string]interface{}) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Errorf("decode rerank request: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
*lastRequest = req
w.Header().Set("Content-Type", "application/json")
if _, truncated := req["truncate_prompt_tokens"]; truncated {
// Query truncated away from the prompt: scores collapse.
_, _ = w.Write([]byte(`{
"id": "rerank-collapsed",
"results": [
{"index": 0, "relevance_score": 0.00670681, "document": {"text": "A"}},
{"index": 1, "relevance_score": 0.00587342, "document": {"text": "B"}},
{"index": 2, "relevance_score": 0.00412907, "document": {"text": "C"}}
],
"usage": {"total_tokens": 42}
}`))
return
}
// Healthy response: sorted by relevance_score descending, index values
// out of order relative to the input documents [A, B, C].
_, _ = w.Write([]byte(`{
"id": "rerank-ok",
"results": [
{"index": 2, "relevance_score": 0.998, "document": {"text": "C"}},
{"index": 0, "relevance_score": 0.51, "document": {"text": "A"}},
{"index": 1, "relevance_score": 0.006, "document": {"text": "B"}}
],
"usage": {"total_tokens": 42}
}`))
}))
}
// TestOpenAIRerankerDoesNotTruncatePromptByDefault is the regression test for
// issue #2143: the generic OpenAI-compatible reranker must not send the
// vLLM-specific truncate_prompt_tokens field unless explicitly configured, and
// each input document must keep its own relevance score via the index field.
func TestOpenAIRerankerDoesNotTruncatePromptByDefault(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
var lastRequest map[string]interface{}
server := newRerankScoreTestServer(t, &lastRequest)
defer server.Close()
reranker, err := NewOpenAIReranker(&RerankerConfig{
BaseURL: server.URL,
ModelName: "Qwen/Qwen3-VL-Reranker-8B",
APIKey: "sk-test",
})
if err != nil {
t.Fatalf("NewOpenAIReranker: %v", err)
}
documents := []string{"A", "B", "C"}
results, err := reranker.Rerank(t.Context(), "query", documents)
if err != nil {
t.Fatalf("Rerank: %v", err)
}
if v, ok := lastRequest["truncate_prompt_tokens"]; ok {
t.Errorf("request contains truncate_prompt_tokens=%v; it must not be sent unless configured", v)
}
if v, ok := lastRequest["additional_data"]; ok {
t.Errorf("request contains additional_data=%v; empty optional fields must be omitted", v)
}
if len(results) != len(documents) {
t.Fatalf("got %d results, want %d", len(results), len(documents))
}
// Results stay in the order returned by the API (sorted by score desc).
if results[0].Index != 2 || results[0].RelevanceScore != 0.998 {
t.Errorf("top result = {index: %d, score: %v}, want {index: 2, score: 0.998}",
results[0].Index, results[0].RelevanceScore)
}
// Each input document keeps its own score, resolved through the index field.
wantScoreByDoc := map[string]float64{"C": 0.998, "A": 0.51, "B": 0.006}
for _, rr := range results {
if rr.Index < 0 || rr.Index >= len(documents) {
t.Fatalf("result index %d out of range for %d documents", rr.Index, len(documents))
}
doc := documents[rr.Index]
if want := wantScoreByDoc[doc]; rr.RelevanceScore != want {
t.Errorf("document %q got score %v, want %v", doc, rr.RelevanceScore, want)
}
}
}
// TestOpenAIRerankerTruncatePromptTokensOptIn verifies that deployments which
// really need server-side prompt truncation (self-hosted vLLM with small-
// context rerankers) can still opt in via extra_config.
func TestOpenAIRerankerTruncatePromptTokensOptIn(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
var lastRequest map[string]interface{}
server := newRerankScoreTestServer(t, &lastRequest)
defer server.Close()
reranker, err := NewOpenAIReranker(&RerankerConfig{
BaseURL: server.URL,
ModelName: "bge-reranker-base",
APIKey: "sk-test",
ExtraConfig: map[string]string{"truncate_prompt_tokens": "511"},
})
if err != nil {
t.Fatalf("NewOpenAIReranker: %v", err)
}
if _, err := reranker.Rerank(t.Context(), "query", []string{"A", "B", "C"}); err != nil {
t.Fatalf("Rerank: %v", err)
}
v, ok := lastRequest["truncate_prompt_tokens"]
if !ok {
t.Fatal("request is missing truncate_prompt_tokens despite extra_config opt-in")
}
if n, _ := v.(float64); n != 511 {
t.Errorf("truncate_prompt_tokens = %v, want 511", v)
}
}
func TestNewOpenAIRerankerRejectsInvalidTruncatePromptTokens(t *testing.T) {
for _, raw := range []string{"abc", "-1", "0"} {
_, err := NewOpenAIReranker(&RerankerConfig{
ModelName: "rerank-test",
ExtraConfig: map[string]string{"truncate_prompt_tokens": raw},
})
if err == nil {
t.Errorf("NewOpenAIReranker accepted invalid truncate_prompt_tokens %q", raw)
}
}
}

View File

@ -4,14 +4,18 @@ import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/provider"
// provider.DetectProvider answers from the vendor catalog, which is empty
// until the vendor packages have run their init. Without this import every
// URL detects as "generic" and newReranker below falls through to the
// OpenAI-compatible reranker for LKEAP / Volcengine / Jina / … rows that
// carry no explicit provider id.
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/api/cohererank"
"github.com/Tencent/WeKnora/internal/models/api/dashscoperank"
"github.com/Tencent/WeKnora/internal/models/api/nimrerank"
"github.com/Tencent/WeKnora/internal/models/catalog"
// catalog.Resolve answers from the vendor catalog, which is empty until
// the vendor packages have run their init. Without this import every row
// resolves to the generic vendor: LKEAP and Volcengine would lose their
// signed clients and Aliyun its native protocol.
_ "github.com/Tencent/WeKnora/internal/models/vendors"
"github.com/Tencent/WeKnora/internal/types"
)
@ -131,45 +135,99 @@ func NewReranker(config *RerankerConfig) (Reranker, error) {
return wrapRerankerLangfuse(r, nil)
}
// customHeaderSetter 表示支持注入自定义 HTTP header 的 reranker 实现。
type customHeaderSetter interface {
SetCustomHeaders(map[string]string)
}
// newReranker resolves the catalog and returns the protocol client for the
// configured model, wrapped in the shared batching and score-scaling layer.
// It mirrors chat.NewRemoteChat: the vendor's facts decide the protocol, the
// URL and the credential, and this function knows no vendor names.
func newReranker(config *RerankerConfig) (Reranker, error) {
// Use provider field if set, otherwise detect from URL using provider registry
providerName := provider.ProviderName(config.Provider)
if providerName == "" {
providerName = provider.DetectProvider(config.BaseURL)
if config == nil {
return nil, fmt.Errorf("rerank config is nil")
}
resolved, err := catalog.Resolve(catalog.Ref{
Provider: config.Provider,
Model: config.ModelName,
BaseURL: config.BaseURL,
ModelType: types.ModelTypeRerank,
Extra: config.ExtraConfig,
})
if err != nil {
return nil, err
}
if err := validateRerankBaseURL(resolved.BaseURL); err != nil {
return nil, err
}
var (
reranker Reranker
err error
)
switch providerName {
case provider.ProviderAliyun:
reranker, err = NewAliyunReranker(config)
case provider.ProviderZhipu:
reranker, err = NewZhipuReranker(config)
case provider.ProviderJina:
reranker, err = NewJinaReranker(config)
case provider.ProviderNvidia:
reranker, err = NewNvidiaReranker(config)
case provider.ProviderWeKnoraCloud:
reranker, err = NewWeKnoraCloudReranker(config)
case provider.ProviderLKEAP:
reranker, err = NewLKEAPReranker(config)
case provider.ProviderVolcengine:
reranker, err = NewVolcengineReranker(config)
vendor := resolved.Vendor
creds := catalog.Credentials{APIKey: config.APIKey, AppID: config.AppID, AppSecret: config.AppSecret}
if creds.APIKey == "" {
creds.APIKey = vendor.DefaultAPIKey
}
// A signing vendor with no identity pair would otherwise send unsigned
// requests and fail at the far end, which is a worse error than this one.
if vendor.Auth == catalog.AuthSigned {
if creds.AppID == "" {
return nil, fmt.Errorf("%s rerank: AppID is required", vendor.Name)
}
if creds.AppSecret == "" {
return nil, fmt.Errorf("%s rerank: AppSecret is required", vendor.Name)
}
}
// Vendor headers first so a user header cannot silently replace a vendor
// beta flag, matching chat.NewRemoteChat.
headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders))
for k, v := range vendor.Headers {
headers[k] = v
}
for k, v := range config.CustomHeaders {
headers[k] = v
}
endpoint := api.Endpoint{
BaseURL: resolved.BaseURL,
Model: resolved.RemoteModel,
ModelID: config.ModelID,
Auth: vendor.AuthFunc(vendor.API, creds),
Headers: headers,
// Most vendors have always run without a client deadline here and let
// the caller's context govern; the ones that declare a timeout get it.
Client: newRerankHTTPClient(
time.Duration(resolved.Rerank.RequestTimeout) * time.Second,
),
}
if vendor.Endpoint != nil {
if url, query := vendor.Endpoint(catalog.EndpointRequest{
BaseURL: resolved.BaseURL,
Model: resolved.RemoteModel,
ModelType: types.ModelTypeRerank,
Extra: config.ExtraConfig,
}); url != "" {
endpoint.URL, endpoint.Query = url, query
}
}
var client api.Reranker
switch resolved.RerankAPI {
case api.RerankCohere:
client = cohererank.New(cohererank.Config{Endpoint: endpoint, Settings: resolved.Rerank})
case api.RerankDashScope:
client = dashscoperank.New(dashscoperank.Config{Endpoint: endpoint, Settings: resolved.Rerank})
case api.RerankNIM:
client = nimrerank.New(nimrerank.Config{Endpoint: endpoint, Settings: resolved.Rerank})
case api.RerankTencentLKEAP:
client, err = newLKEAPClient(config, resolved)
case api.RerankVolcengineKnowledge:
client, err = newVolcengineClient(config, resolved)
default:
reranker, err = NewOpenAIReranker(config)
return nil, fmt.Errorf("unsupported rerank api %q for provider %s", resolved.RerankAPI, vendor.ID)
}
if err != nil {
return nil, err
}
if setter, ok := reranker.(customHeaderSetter); ok {
setter.SetCustomHeaders(config.CustomHeaders)
}
return reranker, nil
return &protocolReranker{
inner: client,
settings: resolved.Rerank,
endpoint: resolved.BaseURL,
modelName: config.ModelName,
modelID: config.ModelID,
}, nil
}

View File

@ -0,0 +1,152 @@
package rerank
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// The two SDK-backed vendors have no protocol package, so their outbound
// shape is not covered by a golden test. These pin what the SDK actually puts
// on the wire, which is what the per-client tests did before the migration.
const volcengineRerankPath = "/api/knowledge/service/rerank"
func TestVolcengineClientOutboundShape(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
var request struct {
Datas []struct {
Query string `json:"query"`
Content *string `json:"content"`
} `json:"datas"`
RerankModel *string `json:"rerank_model"`
RerankInstruction *string `json:"rerank_instruction"`
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, volcengineRerankPath, r.URL.Path)
// The access key identifies the caller in the signature; the secret
// signs it and must never travel.
assert.Contains(t, r.Header.Get("Authorization"), "AKLT-test")
assert.NotContains(t, r.Header.Get("Authorization"), "secret-test")
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.91,0.27]}}`))
}))
defer server.Close()
client, err := newVolcengineClient(
&RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"},
&catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"},
)
require.NoError(t, err)
got, err := client.Rerank(context.Background(), "保留对话数据吗", []string{"会保留", "不会保留"})
require.NoError(t, err)
require.Len(t, got, 2)
assert.Equal(t, 0, got[0].Index)
assert.Equal(t, "会保留", got[0].Text)
assert.InDelta(t, 0.91, got[0].Score, 1e-4)
assert.Equal(t, 1, got[1].Index)
assert.InDelta(t, 0.27, got[1].Score, 1e-4)
require.NotNil(t, request.RerankModel)
assert.Equal(t, "doubao-seed-rerank", *request.RerankModel)
require.Len(t, request.Datas, 2)
assert.Equal(t, "保留对话数据吗", request.Datas[0].Query)
require.NotNil(t, request.Datas[0].Content)
assert.Equal(t, "会保留", *request.Datas[0].Content)
require.NotNil(t, request.RerankInstruction)
assert.Contains(t, *request.RerankInstruction, "Document")
}
func TestVolcengineClientSurfacesAnInBodyErrorCode(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"code":1000030,"message":"quota exceeded","data":{}}`))
}))
defer server.Close()
client, err := newVolcengineClient(
&RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"},
&catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"},
)
require.NoError(t, err)
_, err = client.Rerank(context.Background(), "q", []string{"d"})
require.Error(t, err)
assert.Contains(t, err.Error(), "quota exceeded")
}
// A score list that does not line up with the documents cannot be mapped back
// onto them, so it is an error rather than a partially filled result.
func TestVolcengineClientRejectsAScoreCountMismatch(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"code":0,"data":{"scores":[0.5]}}`))
}))
defer server.Close()
client, err := newVolcengineClient(
&RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"},
&catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"},
)
require.NoError(t, err)
_, err = client.Rerank(context.Background(), "q", []string{"d0", "d1"})
require.Error(t, err)
assert.Contains(t, err.Error(), "score count mismatch")
}
func TestVolcengineClientRequiresAKAndSK(t *testing.T) {
for _, cfg := range []*RerankerConfig{
{AppSecret: "sk"},
{APIKey: "ak"},
} {
_, err := newVolcengineClient(cfg, &catalog.Resolved{BaseURL: "https://example.invalid"})
require.Error(t, err)
assert.Contains(t, err.Error(), "access key and secret key are required")
}
}
func TestLKEAPClientRequiresCredentials(t *testing.T) {
for _, cfg := range []*RerankerConfig{
{AppSecret: "sk"},
{APIKey: "id"},
} {
_, err := newLKEAPClient(cfg, &catalog.Resolved{})
require.Error(t, err)
assert.Contains(t, err.Error(), "secret_id and secret_key are required")
}
}
// The secret key has always been accepted from extra_config as well, because
// the model editor stores it there for this vendor.
func TestLKEAPClientTakesTheSecretKeyFromExtraConfig(t *testing.T) {
client, err := newLKEAPClient(
&RerankerConfig{APIKey: "AKIDxxx", ExtraConfig: map[string]string{"secret_key": "sk"}},
&catalog.Resolved{RemoteModel: "lke-reranker-base"},
)
require.NoError(t, err)
require.NotNil(t, client)
}
func TestLKEAPClientFallsBackToTheDefaultModel(t *testing.T) {
client, err := newLKEAPClient(
&RerankerConfig{APIKey: "AKIDxxx", AppSecret: "sk"},
&catalog.Resolved{},
)
require.NoError(t, err)
inner, ok := client.(*lkeapClient)
require.True(t, ok)
assert.Equal(t, LKEAPDefaultRerankModel, inner.modelName)
assert.False(t, strings.Contains(inner.modelName, " "))
}

View File

@ -16,19 +16,19 @@ func withRerankSSRFWhitelist(t *testing.T, raw string) {
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
}
func TestOpenAIRerankerRejectsInternalBaseURL(t *testing.T) {
func TestRerankerRejectsInternalBaseURL(t *testing.T) {
withRerankSSRFWhitelist(t, "")
_, err := NewOpenAIReranker(&RerankerConfig{
_, err := NewReranker(&RerankerConfig{
BaseURL: "http://169.254.169.254/latest/meta-data/",
ModelName: "rerank-test",
})
if err == nil {
t.Fatalf("NewOpenAIReranker returned nil error for blocked internal BaseURL")
t.Fatalf("NewReranker returned nil error for blocked internal BaseURL")
}
}
func TestOpenAIRerankerBlocksRedirectToInternalURL(t *testing.T) {
func TestRerankerBlocksRedirectToInternalURL(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -36,13 +36,13 @@ func TestOpenAIRerankerBlocksRedirectToInternalURL(t *testing.T) {
}))
defer server.Close()
reranker, err := NewOpenAIReranker(&RerankerConfig{
reranker, err := NewReranker(&RerankerConfig{
BaseURL: server.URL,
ModelName: "rerank-test",
APIKey: "sk-test",
})
if err != nil {
t.Fatalf("NewOpenAIReranker: %v", err)
t.Fatalf("NewReranker: %v", err)
}
_, err = reranker.Rerank(t.Context(), "query", []string{"doc"})

View File

@ -6,37 +6,33 @@ import (
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/api"
"github.com/Tencent/WeKnora/internal/models/catalog"
"github.com/Tencent/WeKnora/internal/models/provider"
"github.com/volcengine/vikingdb-go-sdk/knowledge"
knowledgemodel "github.com/volcengine/vikingdb-go-sdk/knowledge/model"
"golang.org/x/sync/errgroup"
)
const (
// VolcengineRerankBaseURL is the managed Knowledge Service host.
VolcengineRerankBaseURL = provider.VolcengineRerankBaseURL
volcengineRerankPath = "/api/knowledge/service/rerank"
volcengineRerankDefaultModel = "doubao-seed-rerank"
volcengineRerankDefaultRegion = "cn-beijing"
volcengineRerankDefaultInstruction = "Whether the Document answers the Query or matches the content retrieval intent"
volcengineRerankMaxDocuments = 50
// volcengineRerankMaxConcurrency bounds the number of in-flight batch
// requests when the candidate set exceeds volcengineRerankMaxDocuments, so a
// very large embedding_top_k cannot fan out into an unbounded burst of calls.
volcengineRerankMaxConcurrency = 4
)
// VolcengineReranker calls the managed Knowledge Service Rerank API with AK/SK signing.
type VolcengineReranker struct {
// volcengineClient calls the managed Knowledge Service rerank through the
// vikingdb SDK, which owns the AK/SK signing. It implements api.Reranker and
// nothing else: the 50-document ceiling and the batch concurrency are
// declared on the vendor and enforced by protocolReranker.
type volcengineClient struct {
modelName string
instruction string
modelID string
endpoint string
client *knowledge.Client
}
func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error) {
func newVolcengineClient(config *RerankerConfig, resolved *catalog.Resolved) (api.Reranker, error) {
accessKey := strings.TrimSpace(config.APIKey)
secretKey := strings.TrimSpace(config.AppSecret)
if secretKey == "" && config.ExtraConfig != nil {
@ -46,15 +42,12 @@ func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error)
return nil, fmt.Errorf("access key and secret key are required for Volcengine rerank")
}
baseURL := strings.TrimRight(strings.TrimSpace(config.BaseURL), "/")
baseURL := resolved.BaseURL
if baseURL == "" {
baseURL = VolcengineRerankBaseURL
}
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
modelName := strings.TrimSpace(config.ModelName)
modelName := strings.TrimSpace(resolved.RemoteModel)
if modelName == "" {
modelName = volcengineRerankDefaultModel
}
@ -80,70 +73,17 @@ func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error)
if err != nil {
return nil, fmt.Errorf("create Volcengine rerank client: %w", err)
}
return &VolcengineReranker{
modelName: modelName,
instruction: instruction,
modelID: config.ModelID,
endpoint: baseURL,
client: client,
}, nil
return &volcengineClient{modelName: modelName, instruction: instruction, client: client}, nil
}
func (r *VolcengineReranker) Rerank(
// Rerank scores one batch. Every document is paired with the same query and
// instruction, and the reply is a score list in input order.
func (r *volcengineClient) Rerank(
ctx context.Context, query string, documents []string,
) ([]RankResult, error) {
if len(documents) == 0 {
return []RankResult{}, nil
}
// The managed Knowledge Service Rerank API rejects requests carrying more
// than volcengineRerankMaxDocuments items. Upstream callers (chat pipeline,
// agent knowledge search, message search) feed in every retrieval candidate
// and do not cap the count per provider, so a large embedding_top_k or a
// multi-target search can exceed the limit. Each Data item is scored
// independently against the same (query, instruction) pair, so the scores
// are comparable across requests — we can split the documents into limit-
// sized batches, rerank them concurrently, and merge without losing any
// candidate (unlike truncation) or biasing the ranking.
results := make([]RankResult, len(documents))
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(volcengineRerankMaxConcurrency)
for start := 0; start < len(documents); start += volcengineRerankMaxDocuments {
start := start
end := min(start+volcengineRerankMaxDocuments, len(documents))
g.Go(func() error {
scores, err := r.rerankBatch(gctx, query, documents[start:end])
if err != nil {
return err
}
for i, score := range scores {
results[start+i] = RankResult{
Index: start + i,
Document: DocumentInfo{Text: documents[start+i]},
RelevanceScore: score,
}
}
return nil
})
}
if err := g.Wait(); err != nil {
return nil, err
}
return results, nil
}
// rerankBatch scores a single batch of documents (already sized within the API
// limit) and returns the per-document relevance scores in input order.
func (r *VolcengineReranker) rerankBatch(
ctx context.Context, query string, documents []string,
) ([]float64, error) {
) ([]api.RerankResult, error) {
data := make([]knowledgemodel.RerankDataItem, len(documents))
for i := range documents {
data[i] = knowledgemodel.RerankDataItem{
Query: query,
Content: &documents[i],
}
data[i] = knowledgemodel.RerankDataItem{Query: query, Content: &documents[i]}
}
request := knowledgemodel.RerankRequest{
Datas: data,
@ -151,11 +91,6 @@ func (r *VolcengineReranker) rerankBatch(
RerankInstruction: &r.instruction,
}
logger.Debugf(
ctx,
"%s",
buildRerankRequestDebug(r.modelName, r.endpoint+volcengineRerankPath, query, documents),
)
response, err := r.client.Rerank(ctx, request)
if err != nil {
return nil, fmt.Errorf("call Volcengine rerank: %w", err)
@ -169,17 +104,13 @@ func (r *VolcengineReranker) rerankBatch(
if len(response.Data.Scores) != len(documents) {
return nil, fmt.Errorf(
"Volcengine rerank score count mismatch: got %d scores for %d documents",
len(response.Data.Scores),
len(documents),
len(response.Data.Scores), len(documents),
)
}
return response.Data.Scores, nil
}
func (r *VolcengineReranker) GetModelName() string {
return r.modelName
}
func (r *VolcengineReranker) GetModelID() string {
return r.modelID
results := make([]api.RerankResult, len(documents))
for i, score := range response.Data.Scores {
results[i] = api.RerankResult{Index: i, Score: score, Text: documents[i]}
}
return results, nil
}

View File

@ -1,178 +0,0 @@
package rerank
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestVolcengineReranker_Rerank(t *testing.T) {
withRerankSSRFWhitelist(t, "127.0.0.1")
var request struct {
Datas []struct {
Query string `json:"query"`
Content *string `json:"content"`
} `json:"datas"`
RerankModel *string `json:"rerank_model"`
RerankInstruction *string `json:"rerank_instruction"`
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, volcengineRerankPath, r.URL.Path)
assert.Contains(t, r.Header.Get("Authorization"), "AKLT-test")
assert.NotContains(t, r.Header.Get("Authorization"), "secret-test")
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.91,0.27]}}`))
}))
defer server.Close()
reranker, err := NewVolcengineReranker(&RerankerConfig{
APIKey: "AKLT-test",
AppSecret: "secret-test",
BaseURL: server.URL,
ModelName: "doubao-seed-rerank",
ModelID: "volc-rerank",
})
require.NoError(t, err)
results, err := reranker.Rerank(t.Context(), "保留对话数据吗", []string{"会保留", "不会保留"})
require.NoError(t, err)
require.Len(t, results, 2)
assert.Equal(t, 0, results[0].Index)
assert.Equal(t, "会保留", results[0].Document.Text)
assert.InDelta(t, 0.91, results[0].RelevanceScore, 0.0001)
assert.Equal(t, 1, results[1].Index)
assert.Equal(t, "不会保留", results[1].Document.Text)
assert.InDelta(t, 0.27, results[1].RelevanceScore, 0.0001)
require.NotNil(t, request.RerankModel)
assert.Equal(t, "doubao-seed-rerank", *request.RerankModel)
require.Len(t, request.Datas, 2)
assert.Equal(t, "保留对话数据吗", request.Datas[0].Query)
require.NotNil(t, request.Datas[0].Content)
assert.Equal(t, "会保留", *request.Datas[0].Content)
require.NotNil(t, request.RerankInstruction)
assert.True(t, strings.Contains(*request.RerankInstruction, "Document"))
}
func TestNewVolcengineReranker_RequiresAKSK(t *testing.T) {
_, err := NewVolcengineReranker(&RerankerConfig{
APIKey: "ark-api-key-only",
BaseURL: VolcengineRerankBaseURL,
ModelName: "doubao-seed-rerank",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "access key and secret key")
}
func newTestVolcengineReranker(t *testing.T, handler http.HandlerFunc) *VolcengineReranker {
t.Helper()
withRerankSSRFWhitelist(t, "127.0.0.1")
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
reranker, err := NewVolcengineReranker(&RerankerConfig{
APIKey: "AKLT-test",
AppSecret: "secret-test",
BaseURL: server.URL,
ModelName: "doubao-seed-rerank",
})
require.NoError(t, err)
return reranker
}
func TestVolcengineReranker_EmptyDocuments(t *testing.T) {
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
t.Fatalf("rerank endpoint should not be called for empty documents")
})
results, err := reranker.Rerank(t.Context(), "query", nil)
require.NoError(t, err)
assert.Empty(t, results)
}
// TestVolcengineReranker_BatchesOverLimit verifies that a candidate set larger
// than the API document limit is split into batches and reranked in full,
// rather than truncated: every document gets a score and no batch exceeds the
// per-request limit.
func TestVolcengineReranker_BatchesOverLimit(t *testing.T) {
var (
mu sync.Mutex
batchSizes []int
requestCount int
)
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
var body struct {
Datas []struct {
Content *string `json:"content"`
} `json:"datas"`
}
require.NoError(t, json.NewDecoder(r.Body).Decode(&body))
mu.Lock()
requestCount++
batchSizes = append(batchSizes, len(body.Datas))
mu.Unlock()
scores := make([]string, len(body.Datas))
for i := range scores {
scores[i] = "0.5"
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[` +
strings.Join(scores, ",") + `]}}`))
})
total := volcengineRerankMaxDocuments*2 + 10
documents := make([]string, total)
for i := range documents {
documents[i] = fmt.Sprintf("doc-%d", i)
}
results, err := reranker.Rerank(t.Context(), "query", documents)
require.NoError(t, err)
// All documents reranked, each mapped back to its original index/text.
require.Len(t, results, total)
for i := range results {
assert.Equal(t, i, results[i].Index)
assert.Equal(t, documents[i], results[i].Document.Text)
}
// Split into ceil(total/limit) batches, none exceeding the API limit.
assert.Equal(t, 3, requestCount)
for _, size := range batchSizes {
assert.LessOrEqual(t, size, volcengineRerankMaxDocuments)
}
}
func TestVolcengineReranker_APIErrorCode(t *testing.T) {
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":100004,"message":"quota exceeded","data":{}}`))
})
_, err := reranker.Rerank(t.Context(), "query", []string{"a", "b"})
require.Error(t, err)
assert.Contains(t, err.Error(), "100004")
assert.Contains(t, err.Error(), "quota exceeded")
}
func TestVolcengineReranker_ScoreCountMismatch(t *testing.T) {
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.9]}}`))
})
_, err := reranker.Rerank(t.Context(), "query", []string{"a", "b"})
require.Error(t, err)
assert.Contains(t, err.Error(), "score count mismatch")
}

View File

@ -1,134 +0,0 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/google/uuid"
)
const weKnoraCloudRerankPath = "/api/v1/rerank"
// WeKnoraCloudReranker 实现 rerank.Reranker 接口,对接 WeKnoraCloud /api/v1/rerank
type WeKnoraCloudReranker struct {
modelName string
remoteModelName string
modelID string
appID string
apiKey string
baseURL string
client *http.Client
}
// NewWeKnoraCloudReranker 构造 WeKnoraCloudReranker
func NewWeKnoraCloudReranker(config *RerankerConfig) (*WeKnoraCloudReranker, error) {
if config.AppID == "" {
return nil, fmt.Errorf("WeKnoraCloud reranker: AppID is required")
}
if config.AppSecret == "" {
return nil, fmt.Errorf("WeKnoraCloud reranker: AppSecret is required")
}
baseURL := strings.TrimRight(config.BaseURL, "/")
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
remoteModelName := ""
if config.ExtraConfig != nil {
remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"])
}
return &WeKnoraCloudReranker{
modelName: config.ModelName,
remoteModelName: remoteModelName,
modelID: config.ModelID,
appID: config.AppID,
apiKey: config.AppSecret,
baseURL: baseURL,
client: newRerankHTTPClient(60 * time.Second),
}, nil
}
type weKnoraCloudRerankRequest struct {
Model string `json:"model"`
Query string `json:"query"`
Documents []string `json:"documents"`
}
type weKnoraCloudRerankResponse struct {
Results []struct {
Index int `json:"index"`
RelevanceScore float64 `json:"relevance_score"`
Document struct {
Text string `json:"text"`
} `json:"document"`
} `json:"results"`
}
func (r *WeKnoraCloudReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
reqBody := weKnoraCloudRerankRequest{
Model: r.effectiveModelName(),
Query: query,
Documents: documents,
}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("weknoracloud reranker: marshal: %w", err)
}
requestID := uuid.New().String()
headers := utils.Sign(r.appID, r.apiKey, requestID, string(bodyBytes))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.baseURL+weKnoraCloudRerankPath, bytes.NewReader(bodyBytes))
if err != nil {
return nil, fmt.Errorf("weknoracloud reranker: create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
for k, v := range headers {
req.Header.Set(k, v)
}
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("weknoracloud reranker: do request: %w", err)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("weknoracloud reranker: read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("weknoracloud reranker: status %d: %s", resp.StatusCode, string(respBytes))
}
var rerankResp weKnoraCloudRerankResponse
if err := json.Unmarshal(respBytes, &rerankResp); err != nil {
return nil, fmt.Errorf("weknoracloud reranker: unmarshal: %w", err)
}
results := make([]RankResult, 0, len(rerankResp.Results))
for _, item := range rerankResp.Results {
results = append(results, RankResult{
Index: item.Index,
RelevanceScore: item.RelevanceScore,
Document: DocumentInfo{Text: item.Document.Text},
})
}
return results, nil
}
func (r *WeKnoraCloudReranker) effectiveModelName() string {
if r.remoteModelName != "" {
return r.remoteModelName
}
return r.modelName
}
func (r *WeKnoraCloudReranker) GetModelName() string { return r.modelName }
func (r *WeKnoraCloudReranker) GetModelID() string { return r.modelID }

View File

@ -1,153 +0,0 @@
package rerank
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// ZhipuReranker implements a reranking system based on Zhipu AI models
type ZhipuReranker struct {
modelName string // Name of the model used for reranking
modelID string // Unique identifier of the model
apiKey string // API key for authentication
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
customHeaders map[string]string
}
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
func (r *ZhipuReranker) SetCustomHeaders(headers map[string]string) {
r.customHeaders = headers
}
// ZhipuRerankRequest represents a request to rerank documents using Zhipu AI API
type ZhipuRerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Query string `json:"query"` // Query text to compare documents against
Documents []string `json:"documents"` // List of document texts to rerank
TopN int `json:"top_n,omitempty"` // Number of top results to return (0 = all)
ReturnDocuments bool `json:"return_documents,omitempty"` // Whether to return documents in response
ReturnRawScores bool `json:"return_raw_scores,omitempty"` // Whether to return raw scores
}
// ZhipuRerankResponse represents the response from Zhipu AI reranking request
type ZhipuRerankResponse struct {
RequestID string `json:"request_id"` // Request ID from client or platform
ID string `json:"id"` // Task order ID from Zhipu platform
Results []ZhipuRankResult `json:"results"` // Ranked results with relevance scores
Usage ZhipuUsage `json:"usage"` // Token usage information
}
// ZhipuRankResult represents a single reranking result from Zhipu AI
type ZhipuRankResult struct {
Index int `json:"index"` // Original index of the document
RelevanceScore float64 `json:"relevance_score"` // Relevance score
Document string `json:"document,omitempty"` // Document text (optional)
}
// ZhipuUsage contains information about token usage in the Zhipu API request
type ZhipuUsage struct {
TotalTokens int `json:"total_tokens"` // Total tokens consumed
PromptTokens int `json:"prompt_tokens"` // Prompt tokens
}
// NewZhipuReranker creates a new instance of Zhipu reranker with the provided configuration
func NewZhipuReranker(config *RerankerConfig) (*ZhipuReranker, error) {
apiKey := config.APIKey
baseURL := "https://open.bigmodel.cn/api/paas/v4/rerank"
if url := config.BaseURL; url != "" {
baseURL = url
}
if err := validateRerankBaseURL(baseURL); err != nil {
return nil, err
}
return &ZhipuReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
}, nil
}
// Rerank performs document reranking based on relevance to the query using Zhipu AI API
func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body
requestBody := &ZhipuRerankRequest{
Model: r.modelName,
Query: query,
Documents: documents,
TopN: 0, // Return all documents
ReturnDocuments: true,
ReturnRawScores: false,
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
// Send the request
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
secutils.ApplyCustomHeaders(req, r.customHeaders)
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
resp, err := r.client.Do(req)
if err != nil {
return nil, fmt.Errorf("do request: %w", err)
}
defer resp.Body.Close()
// Read the response
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("zhipu rerank API error: Http Status: %s, Body: %s", resp.Status, string(body))
}
var response ZhipuRerankResponse
if err := json.Unmarshal(body, &response); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
// Convert Zhipu results to standard RankResult format
results := make([]RankResult, len(response.Results))
for i, zhipuResult := range response.Results {
results[i] = RankResult{
Index: zhipuResult.Index,
Document: DocumentInfo{
Text: zhipuResult.Document,
},
RelevanceScore: zhipuResult.RelevanceScore,
}
}
return results, nil
}
// GetModelName returns the name of the reranking model
func (r *ZhipuReranker) GetModelName() string {
return r.modelName
}
// GetModelID returns the unique identifier of the reranking model
func (r *ZhipuReranker) GetModelID() string {
return r.modelID
}

View File

@ -47,7 +47,7 @@
{"id": "text-embedding-v4", "name": "Tongyi text-embedding-v4", "type": "Embedding", "dimension": 1024, "source": "https://help.aliyun.com/zh/model-studio/embedding"},
{"id": "text-embedding-v3", "name": "Tongyi text-embedding-v3", "type": "Embedding", "dimension": 1024, "source": "https://help.aliyun.com/zh/model-studio/embedding"},
{"id": "tongyi-embedding-vision-plus", "name": "Tongyi Embedding Vision Plus", "type": "Embedding", "dimension": 1152, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api"},
{"id": "qwen3-rerank", "name": "Qwen3 Rerank", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
{"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
{"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}
]
}

View File

@ -58,12 +58,16 @@
// - rerank is a separate DashScope-native endpoint
// (/api/v1/services/rerank/text-rerank/text-rerank).
//
// unverified: the docs place qwen3-rerank on a different path than every
// other rerank model (/compatible-api/v1/reranks instead of the native
// text-rerank path this package defaults to), and WeKnora's DashScope rerank
// client only speaks the native request shape. The single RerankBaseURL is
// kept; qwen3-rerank needs an operator-supplied base URL until the client
// learns the second shape
// qwen3-rerank is a second, incompatible rerank protocol on the same vendor.
// The text-rerank page puts it on /compatible-api/v1/reranks and states
// outright that "两种接口的请求体结构和响应格式不同": its request is flat
// (query / documents at the top level, no input/parameters wrapper) and its
// response carries `results` at the top level with no `output` object. This
// package implements only the native shape that gte-rerank-v2 and
// qwen3.7-text-rerank use, so the entry is marked deprecated: it stays
// resolvable for a row that already names it, but the picker no longer offers
// a model that would be sent to the wrong path and decoded with the wrong
// shape. Serving it needs a fourth rerank protocol package
// (https://help.aliyun.com/zh/model-studio/text-rerank-api).
//
// unverified: no page states whether `prompt_cache_key` is accepted, so the
@ -108,7 +112,7 @@ func init() {
ID: ID,
Name: "Alibaba Cloud DashScope",
Names: map[string]string{"zh-CN": "阿里云 DashScope"},
Description: "qwen-plus, qwen3.8-max, deepseek-v4-pro, text-embedding-v4, qwen3-rerank, etc.",
Description: "qwen-plus, qwen3.8-max, deepseek-v4-pro, text-embedding-v4, gte-rerank-v2, etc.",
Website: "https://bailian.console.aliyun.com",
Icon: icon,
API: api.APIOpenAICompletions,
@ -128,7 +132,16 @@ func init() {
types.ModelTypeRerank,
types.ModelTypeVLLM,
},
RerankAPI: api.RerankDashScope,
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
SendReturnDocs: catalog.Ptr(true),
// 500 documents per request for the native text-rerank models.
// The query (4,000 tokens) and per-document limits are stated in
// tokens, which a rune count cannot express, so they are not
// declared.
MaxDocuments: catalog.Ptr(500),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
// Explicit although it matches the protocol default, because
// this vendor's own parameter table deprecates the other

View File

@ -148,6 +148,9 @@ func init() {
// api-version. A value switches back to the dated
// /openai/deployments/... path.
Placeholder: "leave empty for /openai/v1, or e.g. 2025-04-01-preview",
Placeholders: map[string]string{
"zh-CN": "留空走 /openai/v1,或填如 2025-04-01-preview",
},
},
},
Compat: catalog.VendorCompat{

View File

@ -71,7 +71,49 @@ func init() {
types.ModelTypeVLLM,
types.ModelTypeASR,
},
ExtraFields: []catalog.ExtraField{{
Key: catalog.ExtraScoreScale,
Label: "Rerank score scale",
Labels: map[string]string{"zh-CN": "Rerank 分数标度"},
Type: "select",
Options: []catalog.ExtraFieldOption{
{
Label: "Unbounded score (Qwen3-Reranker class)",
Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
Value: "logit",
},
{
Label: "0..1 relevance (BGE class)",
Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
Value: "probability",
},
},
Placeholder: "match the reranker actually deployed behind this endpoint",
Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
// A gateway serves whatever reranker was deployed behind it, and
// the two families disagree. The vendor default follows the
// documentation; the operator knows which model is actually there.
Required: false,
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}, {
Key: catalog.ExtraTruncatePromptTokens,
Label: "Rerank prompt truncation (vLLM)",
Labels: map[string]string{"zh-CN": "Rerank 提示词截断长度(vLLM)"},
Type: "number",
// Off unless the operator asks: a runtime that does not implement
// the extension rejects the unknown field.
Required: false,
Placeholder: "empty unless the backend rejects long documents",
Placeholders: map[string]string{"zh-CN": "留空,除非后端因文档过长而报错"},
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// Any OpenAI-compatible endpoint an operator points here is most
// often a vLLM or SGLang server, which is where
// truncate_prompt_tokens comes from.
AcceptsTruncatePromptTokens: catalog.Ptr(true),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
MaxTokensField: catalog.Ptr("max_tokens"),
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs),

View File

@ -81,7 +81,55 @@ func init() {
types.ModelTypeVLLM,
types.ModelTypeASR,
},
ExtraFields: []catalog.ExtraField{{
Key: catalog.ExtraScoreScale,
Label: "Rerank score scale",
Labels: map[string]string{"zh-CN": "Rerank 分数标度"},
Type: "select",
Options: []catalog.ExtraFieldOption{
{
Label: "Unbounded score (Qwen3-Reranker class)",
Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
Value: "logit",
},
{
Label: "0..1 relevance (BGE class)",
Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
Value: "probability",
},
},
Placeholder: "match the reranker actually deployed behind this endpoint",
Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
// A gateway serves whatever reranker was deployed behind it, and
// the two families disagree. The vendor default follows the
// documentation; the operator knows which model is actually there.
Required: false,
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}, {
Key: catalog.ExtraTruncatePromptTokens,
Label: "Rerank prompt truncation (vLLM)",
Labels: map[string]string{"zh-CN": "Rerank 提示词截断长度(vLLM)"},
Type: "number",
// Off unless the operator asks: a runtime that does not implement
// the extension rejects the unknown field.
Required: false,
Placeholder: "empty unless the backend rejects long documents",
Placeholders: map[string]string{"zh-CN": "留空,除非后端因文档过长而报错"},
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// GPUStack's built-in backends are vLLM, SGLang, Ascend MindIE and
// VoxBox, so the vLLM rerank extension reaches the model.
AcceptsTruncatePromptTokens: catalog.Ptr(true),
// The field is spelled relevance_score like the Cohere shape, but
// it is not a probability: the rerank API page's own example
// returns 1.951932668685913, -3.7347371578216553 and
// -6.157620906829834. Read as a probability, every negative score
// falls below a relevance threshold tuned for 0..1.
// https://docs.gpustack.ai/2.0/user-guide/rerank-api/
ScoreScale: catalog.Ptr(api.ScoreLogit),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
MaxTokensField: catalog.Ptr("max_tokens"),
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs),

View File

@ -63,6 +63,14 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
},
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// return_documents echoes the text back. Results are matched
// by index, so this is not needed to map them; it is kept
// because it is what this vendor has always been sent.
SendReturnDocs: catalog.Ptr(true),
},
},
Models: catalog.MustParseModels(modelsJSON),
})
}

View File

@ -48,6 +48,10 @@ const ID = "litellm"
// BaseURL is a placeholder the operator replaces with a reachable proxy.
const BaseURL = "http://your_litellm_proxy/v1"
// RerankBaseURL is the proxy root. LiteLLM documents rerank at /rerank
// rather than /v1/rerank, so it does not hang off the chat base URL.
const RerankBaseURL = "http://your_litellm_proxy"
func init() {
catalog.Register(&catalog.Vendor{
ID: ID,
@ -65,13 +69,40 @@ func init() {
DefaultBaseURLs: map[types.ModelType]string{
types.ModelTypeKnowledgeQA: BaseURL,
types.ModelTypeEmbedding: BaseURL,
types.ModelTypeRerank: RerankBaseURL,
types.ModelTypeVLLM: BaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
},
ExtraFields: []catalog.ExtraField{{
Key: catalog.ExtraScoreScale,
Label: "Rerank score scale",
Labels: map[string]string{"zh-CN": "Rerank 分数标度"},
Type: "select",
Options: []catalog.ExtraFieldOption{
{
Label: "Unbounded score (Qwen3-Reranker class)",
Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
Value: "logit",
},
{
Label: "0..1 relevance (BGE class)",
Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
Value: "probability",
},
},
Placeholder: "match the reranker actually deployed behind this endpoint",
Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
// A gateway serves whatever reranker was deployed behind it, and
// the two families disagree. The vendor default follows the
// documentation; the operator knows which model is actually there.
Required: false,
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
OpenAICompletions: catalog.OpenAICompletionsCompat{
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatOpenAI),

View File

@ -111,8 +111,11 @@ func init() {
Type: "password",
Required: true,
Placeholder: "Tencent Cloud SecretKey (the API Key field holds the SecretId)",
ModelTypes: rerankOnly,
Secret: true,
Placeholders: map[string]string{
"zh-CN": "腾讯云 SecretKey(API Key 那一栏填的是 SecretId)",
},
ModelTypes: rerankOnly,
Secret: true,
},
{
// RunRerank is only published in ap-beijing and ap-guangzhou.
@ -129,7 +132,19 @@ func init() {
ModelTypes: rerankOnly,
},
},
RerankAPI: api.RerankTencentLKEAP,
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// RunRerank takes at most 60 documents, and Query plus Docs
// together at most 2000 characters.
MaxDocuments: catalog.Ptr(60),
MaxRequestChars: catalog.Ptr(2000),
// Batches went out one at a time before the shared batching
// layer existed. A 2000-character budget splits a large
// candidate set into many requests, so the default fan-out of
// four would be a new burst against RunRerank.
MaxConcurrency: catalog.Ptr(1),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
MaxTokensField: catalog.Ptr("max_tokens"),
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),

View File

@ -1,8 +1,14 @@
// Package novita registers the Novita AI gateway.
//
// Facts (https://docs.novita.ai — novita.ai/docs/* redirects there):
// - chat, embedding and VLM share https://api.novita.ai/openai/v1;
// /v1/rerank, /v1/completions, /v1/batches and /v1/files sit alongside;
// - chat, embedding, VLM and rerank share https://api.novita.ai/openai/v1;
// /v1/completions, /v1/batches and /v1/files sit alongside;
// - rerank is the Cohere shape — {model, query, documents, top_n} answering
// results[].{index, relevance_score, document.text} — so it needs no
// vendor settings beyond the type being open
// (https://docs.novita.ai/api-reference/model-apis-llm-create-rerank).
// No ceiling on the document count is documented, and the reference
// lists no model ids, so none are shipped: the operator names the model;
// - output cap is `max_tokens`, documented as required;
// `max_completion_tokens` is never mentioned;
// - the thinking switch is a **top-level** `enable_thinking` boolean
@ -59,11 +65,13 @@ func init() {
DefaultBaseURLs: map[types.ModelType]string{
types.ModelTypeKnowledgeQA: BaseURL,
types.ModelTypeEmbedding: BaseURL,
types.ModelTypeRerank: BaseURL,
types.ModelTypeVLLM: BaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
},
Compat: catalog.VendorCompat{

View File

@ -98,7 +98,17 @@ func init() {
types.ModelTypeRerank,
types.ModelTypeVLLM,
},
RerankAPI: api.RerankNIM,
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// rankings[].logit is unbounded and routinely negative, so the
// caller must be told this is not a 0..1 relevance score.
ScoreScale: catalog.Ptr(api.ScoreLogit),
// truncate defaults to NONE upstream, which fails the request on
// an over-long passage instead of cutting it.
Truncate: catalog.Ptr("END"),
MaxDocuments: catalog.Ptr(512),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
MaxTokensField: catalog.Ptr("max_tokens"),
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs),

View File

@ -18,6 +18,7 @@
{"id": "minimax/minimax-m3", "name": "MiniMax M3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 512000, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.06}},
{"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}},
{"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"}
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
{"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}
]
}

View File

@ -67,11 +67,13 @@ func init() {
DefaultBaseURLs: map[types.ModelType]string{
types.ModelTypeKnowledgeQA: BaseURL,
types.ModelTypeEmbedding: BaseURL,
types.ModelTypeRerank: BaseURL,
types.ModelTypeVLLM: BaseURL,
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
},
Compat: catalog.VendorCompat{

View File

@ -101,6 +101,13 @@ func init() {
types.ModelTypeVLLM,
},
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// "文本数量不超过64"; query "长度不超过1600个字符"; each document
// "长度不超过4096个字符".
MaxDocuments: catalog.Ptr(64),
MaxQueryChars: catalog.Ptr(1600),
MaxDocumentChars: catalog.Ptr(4096),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
// Both spellings are accepted; max_completion_tokens is the
// one that also covers the thinking chain.

View File

@ -90,6 +90,16 @@ func TestEveryCatalogEntryResolves(t *testing.T) {
modelType = types.ModelTypeKnowledgeQA
}
r, err := catalog.Resolve(catalog.Ref{Provider: v.ID, Model: name, ModelType: modelType})
// An entry may declare that this build cannot serve it — a vendor
// whose second rerank dialect has no protocol package. Refusing is
// the point: the alternative is a request shaped for the wrong
// protocol. Such an entry must refuse, and must do it with a reason.
if bytes.Contains(m.Compat, []byte("unsupported_reason")) {
if err == nil {
t.Errorf("%s/%s: declares unsupported_reason but still resolves", id, name)
}
continue
}
if err != nil {
t.Errorf("%s/%s: resolve: %v", id, name, err)
continue

View File

@ -150,8 +150,11 @@ func init() {
Type: "password",
Required: true,
Placeholder: "Volcengine IAM secret key (the API Key field holds the access key)",
ModelTypes: rerankOnly,
Secret: true,
Placeholders: map[string]string{
"zh-CN": "火山引擎 IAM Secret Access Key(API Key 那一栏填的是 Access Key ID)",
},
ModelTypes: rerankOnly,
Secret: true,
},
{
Key: "region",
@ -169,10 +172,18 @@ func init() {
Type: "string",
Default: "Whether the Document answers the Query or matches the content retrieval intent",
Placeholder: "Instruction passed to the rerank model",
ModelTypes: rerankOnly,
Placeholders: map[string]string{
"zh-CN": "传给重排模型的 instruction",
},
ModelTypes: rerankOnly,
},
},
RerankAPI: api.RerankVolcengineKnowledge,
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
MaxDocuments: catalog.Ptr(50),
MaxConcurrency: catalog.Ptr(4),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
SupportsReasoningEffort: catalog.Ptr(true),

View File

@ -90,6 +90,13 @@ func init() {
types.ModelTypeVLLM,
},
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
Path: catalog.Ptr("/api/v1/rerank"),
// The only rerank vendor that has ever had a client deadline
// here. Without it a hung endpoint holds the retrieval stage
// for as long as the caller's context allows.
RequestTimeout: catalog.Ptr(60),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
SupportsMultiContent: catalog.Ptr(false),
PromptCacheAccounting: catalog.Ptr(false),

View File

@ -109,6 +109,14 @@ func init() {
types.ModelTypeVLLM,
},
Compat: catalog.VendorCompat{
Rerank: catalog.RerankCompat{
// The reference gives 最大长度为 4096 字符 for the query and for each
// document, and caps documents at 128 per request.
SendReturnDocs: catalog.Ptr(true),
MaxDocuments: catalog.Ptr(128),
MaxQueryChars: catalog.Ptr(4096),
MaxDocumentChars: catalog.Ptr(4096),
},
OpenAICompletions: catalog.OpenAICompletionsCompat{
MaxTokensField: catalog.Ptr("max_tokens"),
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),