From d6c7665cb581908ebdd236b3b53596b97a63464e Mon Sep 17 00:00:00 2001 From: lyingbug Date: Mon, 21 Sep 2026 10:36:26 +0800 Subject: [PATCH] refactor(rerank): move the wire formats into protocol packages, and correct the vendor facts (#3488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 把 8 个手写 rerank client 收敛成 3 个协议包(internal/models/api/{cohererank,dashscoperank,nimrerank}),每个带一份从厂商文档逐字粘来的 golden fixture。LKEAP 和火山保留官方 SDK,但只实现 api.Reranker。rerank.NewReranker 改成 catalog.Resolve + 按协议分发,函数体里没有厂商名。生产代码净 -147 行。 三件原来每个 client 各管各、现在由共用层统一的事:分批(按厂商声明的上限,智谱 128 / NIM 512 等此前无人执行)、分数标度(NIM 与 GPUStack 返回 logit,其余为 0~1 概率,统一换算后再与 RerankThreshold 比较)、NIM 的 truncate(默认 NONE 会让超长 passage 整请求失败,此前从未发送)。 对照官方文档订正四处厂商事实:GPUStack 分数标度、千帆三项上限、阿里 500 条上限、qwen3-rerank 是第二种协议(隐藏并在构造期拒绝)。Novita / OpenRouter / LiteLLM 三家官方有 rerank 但未开类型,本次开放,零协议代码。 运行时行为变更:NVIDIA 起发 truncate: END;GPUStack 分数按 logit 换算(可按行覆盖);truncate_prompt_tokens 收窄为厂商声明制;qwen3-rerank 存量行从静默打错变为构造期报错。详见 PR 描述。 关联 #3486。 --- frontend/src/api/initialization/index.ts | 2 + frontend/src/components/ModelEditorDialog.vue | 23 +- .../src/components/modelEditorFlow.test.ts | 63 +++ .../src/stores/modelProvidersState.test.ts | 36 ++ frontend/src/stores/modelProvidersState.ts | 17 + internal/models/api/batch.go | 79 +++ internal/models/api/batch_test.go | 74 +++ internal/models/api/cohererank/client.go | 167 ++++++ internal/models/api/cohererank/golden_test.go | 203 +++++++ internal/models/api/dashscoperank/client.go | 116 ++++ .../models/api/dashscoperank/golden_test.go | 116 ++++ internal/models/api/endpoint.go | 34 ++ internal/models/api/nimrerank/client.go | 106 ++++ internal/models/api/nimrerank/golden_test.go | 128 +++++ internal/models/api/rerank.go | 83 +++ internal/models/catalog/compat.go | 3 + internal/models/catalog/legacy_rows_test.go | 9 +- internal/models/catalog/registry.go | 3 + internal/models/catalog/rerank.go | 92 +++ internal/models/catalog/resolve.go | 102 ++++ internal/models/catalog/types.go | 41 +- internal/models/parity/parity_test.go | 8 + internal/models/parity/rerank_test.go | 522 ++++++++++++++++++ internal/models/rerank/aliyun_reranker.go | 172 ------ internal/models/rerank/jina_reranker.go | 128 ----- internal/models/rerank/lkeap_reranker.go | 123 +---- internal/models/rerank/lkeap_reranker_test.go | 178 ------ internal/models/rerank/nvidia_reranker.go | 156 ------ .../models/rerank/nvidia_reranker_test.go | 49 -- internal/models/rerank/protocol.go | 142 +++++ internal/models/rerank/protocol_test.go | 187 +++++++ internal/models/rerank/remote_api.go | 153 ----- internal/models/rerank/remote_api_test.go | 157 ------ internal/models/rerank/reranker.go | 136 +++-- internal/models/rerank/sdk_clients_test.go | 152 +++++ .../models/rerank/transport_security_test.go | 12 +- internal/models/rerank/volcengine_reranker.go | 115 +--- .../models/rerank/volcengine_reranker_test.go | 178 ------ internal/models/rerank/weknoracloud.go | 134 ----- internal/models/rerank/zhipu_reranker.go | 153 ----- internal/models/vendors/aliyun/models.json | 2 +- internal/models/vendors/aliyun/vendor.go | 27 +- .../models/vendors/azure_openai/vendor.go | 3 + internal/models/vendors/generic/vendor.go | 42 ++ internal/models/vendors/gpustack/vendor.go | 48 ++ internal/models/vendors/jina/vendor.go | 8 + internal/models/vendors/litellm/vendor.go | 31 ++ internal/models/vendors/lkeap/vendor.go | 19 +- internal/models/vendors/novita/vendor.go | 12 +- internal/models/vendors/nvidia/vendor.go | 10 + .../models/vendors/openrouter/models.json | 3 +- internal/models/vendors/openrouter/vendor.go | 2 + internal/models/vendors/qianfan/vendor.go | 7 + internal/models/vendors/vendors_test.go | 10 + internal/models/vendors/volcengine/vendor.go | 17 +- .../models/vendors/weknoracloud/vendor.go | 7 + internal/models/vendors/zhipu/vendor.go | 8 + 57 files changed, 2875 insertions(+), 1733 deletions(-) create mode 100644 internal/models/api/batch.go create mode 100644 internal/models/api/batch_test.go create mode 100644 internal/models/api/cohererank/client.go create mode 100644 internal/models/api/cohererank/golden_test.go create mode 100644 internal/models/api/dashscoperank/client.go create mode 100644 internal/models/api/dashscoperank/golden_test.go create mode 100644 internal/models/api/nimrerank/client.go create mode 100644 internal/models/api/nimrerank/golden_test.go create mode 100644 internal/models/api/rerank.go create mode 100644 internal/models/catalog/rerank.go create mode 100644 internal/models/parity/rerank_test.go delete mode 100644 internal/models/rerank/aliyun_reranker.go delete mode 100644 internal/models/rerank/jina_reranker.go delete mode 100644 internal/models/rerank/lkeap_reranker_test.go delete mode 100644 internal/models/rerank/nvidia_reranker.go delete mode 100644 internal/models/rerank/nvidia_reranker_test.go create mode 100644 internal/models/rerank/protocol.go create mode 100644 internal/models/rerank/protocol_test.go delete mode 100644 internal/models/rerank/remote_api.go delete mode 100644 internal/models/rerank/remote_api_test.go create mode 100644 internal/models/rerank/sdk_clients_test.go delete mode 100644 internal/models/rerank/volcengine_reranker_test.go delete mode 100644 internal/models/rerank/weknoracloud.go delete mode 100644 internal/models/rerank/zhipu_reranker.go diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index a7bf1b5c..7fd1c3db 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -584,6 +584,7 @@ export function fabriTag(request: FabriTagRequest): Promise { // Mirrors internal/models/catalog.ExtraField. export interface ModelProviderExtraFieldOption { label: string; + labels?: Record; value: string; } @@ -613,6 +614,7 @@ export interface ModelProviderExtraField { required?: boolean; default?: string; placeholder?: string; + placeholders?: Record; options?: ModelProviderExtraFieldOption[]; // Backend model types ("KnowledgeQA", "Embedding", "Rerank", "VLLM", "ASR"); empty = all. model_types?: string[]; diff --git a/frontend/src/components/ModelEditorDialog.vue b/frontend/src/components/ModelEditorDialog.vue index aa169682..8a1f7a3f 100644 --- a/frontend/src/components/ModelEditorDialog.vue +++ b/frontend/src/components/ModelEditorDialog.vue @@ -358,22 +358,24 @@
- {{ field.placeholder }} + {{ extraFieldDisplayPlaceholder(field) }}
- + - @@ -599,7 +601,8 @@ import { MessagePlugin, DialogPlugin } from 'tdesign-vue-next' import { checkOllamaModels, checkRemoteModel, testEmbeddingModel, checkRerankModel, checkASRModel, listOllamaModels, downloadOllamaModel, getDownloadProgress, checkOllamaStatus, resolveModelCatalog, - type OllamaModelInfo, type ModelProviderOption, type ModelProviderExtraField, type ModelCatalogEntry, + type OllamaModelInfo, type ModelProviderOption, type ModelProviderExtraField, + type ModelProviderExtraFieldOption, type ModelCatalogEntry, type ResolvedModelCatalog, } from '@/api/initialization' import { @@ -615,6 +618,8 @@ import { useModelProvidersStore } from '@/stores/modelProviders' import { credentialLabelForModelType, extraFieldLabel, + extraFieldOptionLabel, + extraFieldPlaceholder, extraFieldsForModelType, pickLocalized, providerDescription, @@ -775,6 +780,10 @@ const matchTriggerWidth = (triggerElement: HTMLElement) => ({ }) const extraFieldDisplayLabel = (field: ModelProviderExtraField) => extraFieldLabel(field, currentLocale.value) +const extraFieldDisplayPlaceholder = (field: ModelProviderExtraField) => + extraFieldPlaceholder(field, currentLocale.value) +const extraFieldDisplayOptionLabel = (option: ModelProviderExtraFieldOption) => + extraFieldOptionLabel(option, currentLocale.value) /** * Vendors whose API is not a bearer-token API name their first credential diff --git a/frontend/src/components/modelEditorFlow.test.ts b/frontend/src/components/modelEditorFlow.test.ts index 6769c73e..764a52f9 100644 --- a/frontend/src/components/modelEditorFlow.test.ts +++ b/frontend/src/components/modelEditorFlow.test.ts @@ -812,3 +812,66 @@ test('a vendor with no catalog falls back to a plain text field', async () => { } finally { f2.close() } } finally { f.close() } }) + +// --- Vendor strings reach the form in the reader's language --------------- + +// The editor renders whatever the catalog declares, so every operator-facing +// string a vendor ships has to go through the locale resolvers. Binding a raw +// `field.placeholder` or `opt.label` is silent: the form still works, it is +// just English in a Chinese product, and a test that only checks the catalog +// for a zh-CN entry passes while the editor ignores it. This reads the +// template instead. +test('every vendor-declared string in the form goes through a locale resolver', () => { + const template = descriptor.template?.content ?? '' + assert.ok(template.length > 0, 'the editor template should have been parsed') + + // `field` is the extra-field loop variable, so any read of its placeholder + // is a vendor string rendered without resolving the locale. One input + // branch per field type plus the secret field means it is easy to bind four + // and miss the fifth, which is exactly what happened. + for (const pattern of [ + /:placeholder="field\.placeholder/, + /\{\{\s*field\.placeholder\s*\}\}/, + /v-if="field\.placeholder"/, + ]) { + assert.equal( + pattern.test(template), false, + `template reads field.placeholder directly (${pattern}); use extraFieldDisplayPlaceholder`, + ) + } + + // Options: scope to the extra-field select, because `opt` is also the loop + // variable of the model-type list and the catalog model list, whose labels + // come from i18n and from the catalog's own names. + const optionLoop = /v-for="opt in \(field\.options \|\| \[\]\)"[^>]*/.exec(template) + assert.ok(optionLoop, 'the extra-field select should loop over field.options') + assert.match( + optionLoop[0], /:label="extraFieldDisplayOptionLabel\(opt\)"/, + 'extra-field option labels must resolve the locale', + ) + + // And the placeholder resolver is referenced, so the checks above cannot be + // satisfied by dropping the placeholders altogether. + assert.ok(template.includes('extraFieldDisplayPlaceholder('), 'placeholders must use the resolver') +}) + +test('extraFieldDisplayPlaceholder resolves the vendor placeholder for the active locale', async () => { + const f = await fixture({ type: 'rerank', providers: catalogProviders }) + try { + const field = { + key: 'score_scale', + label: 'Rerank score scale', + type: 'select', + placeholder: 'match the reranker actually deployed behind this endpoint', + placeholders: { 'zh-CN': '按这个端点后面实际部署的重排模型选择' }, + } + const rendered = f.vm.extraFieldDisplayPlaceholder(field) + assert.ok( + rendered === field.placeholder || rendered === field.placeholders['zh-CN'], + `expected one of the declared variants, got ${rendered}`, + ) + assert.equal(f.vm.extraFieldDisplayPlaceholder({ key: 'k', label: 'k', type: 'string' }), '') + } finally { + f.close() + } +}) diff --git a/frontend/src/stores/modelProvidersState.test.ts b/frontend/src/stores/modelProvidersState.test.ts index 3e8e5dd0..ed1c29e2 100644 --- a/frontend/src/stores/modelProvidersState.test.ts +++ b/frontend/src/stores/modelProvidersState.test.ts @@ -6,6 +6,8 @@ import type { ModelProviderOption } from '@/api/initialization' import { credentialLabelForModelType, extraFieldLabel, + extraFieldOptionLabel, + extraFieldPlaceholder, extraFieldsForModelType, loadProvidersForType, mergeProviderIndex, @@ -222,3 +224,37 @@ test('credentialLabelForModelType follows the vendor declaration, not a hardcode assert.equal(pickLocalized(rerank?.labels, 'zh-CN', rerank!.label), 'SecretId(TC3 签名)') assert.equal(pickLocalized(rerank?.labels, 'en-US', rerank!.label), 'SecretId') }) + +// Select options and placeholders used to render their raw English string. +// That was invisible while every option was an identifier — LKEAP's region +// codes — and became a gap as soon as one was prose. +test('extraFieldOptionLabel resolves the locale, falling back to label then value', () => { + const option = { + label: '0..1 relevance (BGE class)', + labels: { 'zh-CN': '0~1 相关度(BGE 一类)' }, + value: 'probability', + } + assert.equal(extraFieldOptionLabel(option, 'zh-CN'), '0~1 相关度(BGE 一类)') + assert.equal(extraFieldOptionLabel(option, 'en'), '0..1 relevance (BGE class)') + assert.equal( + extraFieldOptionLabel({ label: 'ap-guangzhou', value: 'ap-guangzhou' }, 'zh-CN'), + 'ap-guangzhou', + ) + assert.equal(extraFieldOptionLabel({ value: 'logit' }, 'zh-CN'), 'logit') +}) + +test('extraFieldPlaceholder resolves the locale and tolerates no placeholder', () => { + const field = { + key: 'score_scale', + label: 'Rerank score scale', + type: 'select', + placeholder: 'match the reranker actually deployed behind this endpoint', + placeholders: { 'zh-CN': '按这个端点后面实际部署的重排模型选择' }, + } + assert.equal(extraFieldPlaceholder(field, 'zh-CN'), '按这个端点后面实际部署的重排模型选择') + assert.equal( + extraFieldPlaceholder(field, 'en'), + 'match the reranker actually deployed behind this endpoint', + ) + assert.equal(extraFieldPlaceholder({ key: 'k', label: 'k', type: 'string' }, 'zh-CN'), '') +}) diff --git a/frontend/src/stores/modelProvidersState.ts b/frontend/src/stores/modelProvidersState.ts index 2e0acc39..202b43d9 100644 --- a/frontend/src/stores/modelProvidersState.ts +++ b/frontend/src/stores/modelProvidersState.ts @@ -109,6 +109,23 @@ export function extraFieldLabel(field: ModelProviderExtraField, locale?: string) return pickLocalized(field.labels, locale, field.label || field.key) } +/** Placeholder for the current locale, falling back to the default string. */ +export function extraFieldPlaceholder(field: ModelProviderExtraField, locale?: string): string { + return pickLocalized(field.placeholders, locale, field.placeholder || '') +} + +/** + * Option label for the current locale. Select options used to render their + * raw `label`, which was fine while every option was an identifier (a region + * code) and wrong as soon as one was prose. + */ +export function extraFieldOptionLabel( + option: { label?: string; labels?: Record; value: string }, + locale?: string, +): string { + return pickLocalized(option.labels, locale, option.label || option.value) +} + /** Stable display order: backend `order`, then label. */ export function sortProviders(providers: ReadonlyArray): ModelProviderOption[] { return [...providers].sort((a, b) => { diff --git a/internal/models/api/batch.go b/internal/models/api/batch.go new file mode 100644 index 00000000..e992a7f0 --- /dev/null +++ b/internal/models/api/batch.go @@ -0,0 +1,79 @@ +package api + +import ( + "fmt" + "unicode/utf8" +) + +// BatchLimits are the per-request ceilings a vendor documents. A zero field +// means the vendor states no limit, and nothing is enforced for it. +type BatchLimits struct { + // MaxItems caps how many documents one request may carry. + MaxItems int + // MaxItemRunes caps a single document. Vendors state these in characters + // or tokens; runes are the closest thing we can count without a + // tokenizer, and undercounting is the safe direction. + MaxItemRunes int + // MaxTotalRunes caps the whole request, query included. + MaxTotalRunes int +} + +// Batch is one slice of the input, and where it started. +type Batch struct { + Start int + Items []string +} + +// SplitBatches divides items into requests that satisfy limits. fixedRunes is +// charged to every batch (the query, which each request repeats). +// +// A single item that cannot fit on its own is an error rather than a silent +// truncation: the caller asked for this document to be scored, and scoring a +// prefix of it returns a number for something the user never sent. +func SplitBatches(items []string, fixedRunes int, limits BatchLimits) ([]Batch, error) { + if limits.MaxTotalRunes > 0 && fixedRunes >= limits.MaxTotalRunes { + return nil, fmt.Errorf( + "query is %d characters; the request budget is %d and must also fit a document", + fixedRunes, limits.MaxTotalRunes, + ) + } + + sizes := make([]int, len(items)) + for i, item := range items { + sizes[i] = utf8.RuneCountInString(item) + if limits.MaxItemRunes > 0 && sizes[i] > limits.MaxItemRunes { + return nil, fmt.Errorf( + "document at index %d is %d characters; the per-document limit is %d", + i, sizes[i], limits.MaxItemRunes, + ) + } + if limits.MaxTotalRunes > 0 && fixedRunes+sizes[i] > limits.MaxTotalRunes { + return nil, fmt.Errorf( + "document at index %d is %d characters; it does not fit beside a %d-character query "+ + "in the %d-character request budget", + i, sizes[i], fixedRunes, limits.MaxTotalRunes, + ) + } + } + + if len(items) == 0 { + return nil, nil + } + if limits.MaxItems <= 0 && limits.MaxTotalRunes <= 0 { + return []Batch{{Start: 0, Items: items}}, nil + } + + batches := make([]Batch, 0, 1) + start, total := 0, fixedRunes + for i := range items { + full := limits.MaxItems > 0 && i-start == limits.MaxItems + over := limits.MaxTotalRunes > 0 && total+sizes[i] > limits.MaxTotalRunes + if i > start && (full || over) { + batches = append(batches, Batch{Start: start, Items: items[start:i]}) + start, total = i, fixedRunes + } + total += sizes[i] + } + batches = append(batches, Batch{Start: start, Items: items[start:]}) + return batches, nil +} diff --git a/internal/models/api/batch_test.go b/internal/models/api/batch_test.go new file mode 100644 index 00000000..b70685db --- /dev/null +++ b/internal/models/api/batch_test.go @@ -0,0 +1,74 @@ +package api + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSplitBatchesNoLimitsKeepsOneRequest(t *testing.T) { + got, err := SplitBatches([]string{"a", "b", "c"}, 0, BatchLimits{}) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, 0, got[0].Start) + assert.Len(t, got[0].Items, 3) +} + +func TestSplitBatchesSplitsOnItemCount(t *testing.T) { + items := []string{"a", "b", "c", "d", "e"} + got, err := SplitBatches(items, 0, BatchLimits{MaxItems: 2}) + require.NoError(t, err) + require.Len(t, got, 3) + assert.Equal(t, []Batch{ + {Start: 0, Items: items[0:2]}, + {Start: 2, Items: items[2:4]}, + {Start: 4, Items: items[4:5]}, + }, got) +} + +// The query travels in every request, so it is charged to every batch rather +// than once to the whole call. +func TestSplitBatchesChargesTheQueryToEveryBatch(t *testing.T) { + items := []string{strings.Repeat("x", 30), strings.Repeat("y", 30), strings.Repeat("z", 30)} + got, err := SplitBatches(items, 40, BatchLimits{MaxTotalRunes: 100}) + require.NoError(t, err) + // 40 + 30 + 30 = 100 fits; adding the third would reach 130. + require.Len(t, got, 2) + assert.Len(t, got[0].Items, 2) + assert.Equal(t, 2, got[1].Start) +} + +func TestSplitBatchesRejectsAnItemThatCannotFit(t *testing.T) { + _, err := SplitBatches([]string{"ok", strings.Repeat("x", 50)}, 0, BatchLimits{MaxItemRunes: 10}) + require.Error(t, err) + assert.Contains(t, err.Error(), "index 1") +} + +// A document that fits the per-document limit but not beside the query is +// still unsendable; saying so beats looping forever or silently dropping it. +func TestSplitBatchesRejectsAnItemThatCannotFitBesideTheQuery(t *testing.T) { + _, err := SplitBatches([]string{strings.Repeat("x", 80)}, 40, BatchLimits{MaxTotalRunes: 100}) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not fit beside") +} + +func TestSplitBatchesRejectsAQueryThatFillsTheBudget(t *testing.T) { + _, err := SplitBatches([]string{"a"}, 100, BatchLimits{MaxTotalRunes: 100}) + require.Error(t, err) + assert.Contains(t, err.Error(), "must also fit a document") +} + +// Runes, not bytes: a CJK document is counted the way the vendor counts it. +func TestSplitBatchesCountsRunesNotBytes(t *testing.T) { + got, err := SplitBatches([]string{strings.Repeat("中", 10)}, 0, BatchLimits{MaxItemRunes: 10}) + require.NoError(t, err) + assert.Len(t, got, 1) +} + +func TestSplitBatchesEmptyInput(t *testing.T) { + got, err := SplitBatches(nil, 0, BatchLimits{MaxItems: 5}) + require.NoError(t, err) + assert.Empty(t, got) +} diff --git a/internal/models/api/cohererank/client.go b/internal/models/api/cohererank/client.go new file mode 100644 index 00000000..a8365048 --- /dev/null +++ b/internal/models/api/cohererank/client.go @@ -0,0 +1,167 @@ +// Package cohererank implements the rerank shape Cohere defined and almost +// every other vendor copied: POST {base}/rerank with {model, query, +// documents} answering {results: [{index, relevance_score, document}]}. +// +// Jina, Zhipu, SiliconFlow, Qianfan, GPUStack, WeKnora Cloud and any +// OpenAI-compatible gateway that serves rerank all speak it. Their +// differences — which optional fields are accepted, where the endpoint sits, +// what the documented ceilings are — arrive as catalog.RerankSettings; this +// package contains no vendor names. +package cohererank + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" +) + +// Config is everything the client needs, already resolved by the catalog. +type Config struct { + Endpoint api.Endpoint + Settings catalog.RerankSettings +} + +// Client talks the Cohere rerank shape to one endpoint. +type Client struct { + cfg Config +} + +// New builds a client. +func New(cfg Config) *Client { return &Client{cfg: cfg} } + +const defaultPath = "/rerank" + +// url resolves the endpoint. A vendor whose default base URL already names +// the full rerank endpoint — and an operator who pasted one — must not have +// the path appended twice. +func (c *Client) url() string { + if c.cfg.Endpoint.URL != "" { + return c.cfg.Endpoint.Resolve("") + } + path := c.cfg.Settings.Path + if path == "" { + path = defaultPath + } + if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) { + path = "" + } + return c.cfg.Endpoint.Resolve(path) +} + +type request struct { + Model string `json:"model"` + Query string `json:"query"` + Documents []string `json:"documents"` + TopN int `json:"top_n,omitempty"` + // TruncatePromptTokens is a vLLM extension, not part of any vendor's + // documented schema. It is sent only when the operator opted in, because + // a gateway that does not implement it rejects the unknown field. + TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` + ReturnDocuments *bool `json:"return_documents,omitempty"` +} + +type response struct { + Results []result `json:"results"` +} + +type result struct { + Index int `json:"index"` + RelevanceScore *float64 `json:"relevance_score"` + Score *float64 `json:"score"` + Document document `json:"document"` +} + +// document is the echoed text, which vendors spell either as a bare string +// or as an object with a text field. +type document struct { + Text string +} + +func (d *document) UnmarshalJSON(data []byte) error { + var text string + if err := json.Unmarshal(data, &text); err == nil { + d.Text = text + return nil + } + var obj struct { + Text string `json:"text"` + } + if err := json.Unmarshal(data, &obj); err != nil { + return fmt.Errorf("decode document: %w", err) + } + d.Text = obj.Text + return nil +} + +// BuildRequestBody is the golden-test entry point: it returns the exact JSON +// object that would be sent. +func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) { + body := request{ + Model: c.cfg.Endpoint.Model, + Query: query, + Documents: documents, + } + if c.cfg.Settings.AcceptsTruncatePromptTokens { + body.TruncatePromptTokens = c.cfg.Settings.TruncatePromptTokens + } + if c.cfg.Settings.SendTopN { + body.TopN = len(documents) + } + if c.cfg.Settings.SendReturnDocs { + enabled := true + body.ReturnDocuments = &enabled + } + raw, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + for k, v := range c.cfg.Settings.ExtraBody { + if _, exists := out[k]; !exists { + out[k] = v + } + } + return out, nil +} + +// Rerank scores documents against the query, in the order they were given. +func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) { + body, err := c.BuildRequestBody(query, documents) + if err != nil { + return nil, err + } + var decoded response + if err := c.cfg.Endpoint.PostJSON(ctx, c.url(), body, &decoded); err != nil { + return nil, err + } + out := make([]api.RerankResult, 0, len(decoded.Results)) + for _, item := range decoded.Results { + if item.Index < 0 || item.Index >= len(documents) { + return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents)) + } + out = append(out, api.RerankResult{ + Index: item.Index, + Score: firstNonNil(item.RelevanceScore, item.Score), + Text: item.Document.Text, + }) + } + return out, nil +} + +// firstNonNil picks relevance_score, falling back to score: the field is +// spelled both ways across vendors that otherwise share this shape. +func firstNonNil(values ...*float64) float64 { + for _, v := range values { + if v != nil { + return *v + } + } + return 0 +} diff --git a/internal/models/api/cohererank/golden_test.go b/internal/models/api/cohererank/golden_test.go new file mode 100644 index 00000000..7b2ef8b3 --- /dev/null +++ b/internal/models/api/cohererank/golden_test.go @@ -0,0 +1,203 @@ +package cohererank + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// zhipuResponse is the documented response of Zhipu's rerank endpoint, whose +// result objects carry document / index / relevance_score and whose envelope +// adds id, request_id, created and usage: +// https://docs.bigmodel.cn/api-reference/模型-api/文本重排序 +const zhipuResponse = `{ + "id": "8305283656248111323", + "request_id": "8305283656248111323", + "created": 1769040000, + "results": [ + {"index": 1, "relevance_score": 0.9819, "document": "上海气候"}, + {"index": 0, "relevance_score": 0.0021, "document": "北京美食"} + ], + "usage": {"prompt_tokens": 24, "total_tokens": 24} +}` + +// jinaResponse spells the echoed document as an object instead of a string, +// which is the other shape vendors use for the same field: +// https://jina.ai/reranker/ +const jinaResponse = `{ + "model": "jina-reranker-v3.5", + "results": [ + {"index": 0, "relevance_score": 0.95, "document": {"text": "上海气候"}}, + {"index": 1, "relevance_score": 0.11, "document": {"text": "北京美食"}} + ], + "usage": {"total_tokens": 32} +}` + +func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client { + t.Helper() + return New(Config{ + Endpoint: api.Endpoint{BaseURL: url, Model: "rerank", Auth: api.BearerAuth("k")}, + Settings: settings, + }) +} + +func serve(t *testing.T, payload string) (*httptest.Server, *string, *map[string]any) { + t.Helper() + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + path := new(string) + body := new(map[string]any) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *path = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(payload)) + })) + return server, path, body +} + +func TestRequestBodyCarriesOnlyWhatTheVendorDeclares(t *testing.T) { + c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{}) + body, err := c.BuildRequestBody("上海天气", []string{"上海气候", "北京美食"}) + require.NoError(t, err) + + assert.Equal(t, map[string]any{ + "model": "rerank", + "query": "上海天气", + "documents": []any{"上海气候", "北京美食"}, + }, body, "optional fields must be absent unless the vendor asked for them") +} + +func TestRequestBodyAddsTheOptionalFieldsAVendorDeclares(t *testing.T) { + c := newClient(t, "https://example.invalid/v1", + catalog.RerankSettings{SendTopN: true, SendReturnDocs: true}) + body, err := c.BuildRequestBody("q", []string{"a", "b", "c"}) + require.NoError(t, err) + + assert.Equal(t, float64(3), body["top_n"], "top_n asks for every document back") + assert.Equal(t, true, body["return_documents"]) +} + +func TestDecodesAStringDocument(t *testing.T) { + server, _, _ := serve(t, zhipuResponse) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + got, err := c.Rerank(context.Background(), "上海天气", []string{"北京美食", "上海气候"}) + require.NoError(t, err) + + assert.Equal(t, []api.RerankResult{ + {Index: 1, Score: 0.9819, Text: "上海气候"}, + {Index: 0, Score: 0.0021, Text: "北京美食"}, + }, got) +} + +func TestDecodesAnObjectDocument(t *testing.T) { + server, _, _ := serve(t, jinaResponse) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + got, err := c.Rerank(context.Background(), "上海天气", []string{"上海气候", "北京美食"}) + require.NoError(t, err) + + assert.Equal(t, []api.RerankResult{ + {Index: 0, Score: 0.95, Text: "上海气候"}, + {Index: 1, Score: 0.11, Text: "北京美食"}, + }, got) +} + +// Some gateways serving this shape spell the score `score`. The pre-catalog +// client accepted both and so does this one. +func TestDecodesTheScoreAlias(t *testing.T) { + server, _, _ := serve(t, `{"results":[{"index":0,"score":0.42}]}`) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + got, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.NoError(t, err) + require.Len(t, got, 1) + assert.InDelta(t, 0.42, got[0].Score, 1e-9) +} + +func TestAppendsTheRerankPathToABareBaseURL(t *testing.T) { + server, path, _ := serve(t, `{"results":[]}`) + defer server.Close() + + c := newClient(t, server.URL+"/v1", catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.NoError(t, err) + assert.Equal(t, "/v1/rerank", *path) +} + +// Zhipu's default base URL already ends in /rerank, and operators paste full +// endpoints too. Appending the path again would 404. +func TestDoesNotDoubleTheRerankPath(t *testing.T) { + server, path, _ := serve(t, `{"results":[]}`) + defer server.Close() + + c := newClient(t, server.URL+"/api/paas/v4/rerank", catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.NoError(t, err) + assert.Equal(t, "/api/paas/v4/rerank", *path) +} + +// WeKnora Cloud serves the same shape on its own path. +func TestHonoursAVendorDeclaredPath(t *testing.T) { + server, path, _ := serve(t, `{"results":[]}`) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{Path: "/api/v1/rerank"}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.NoError(t, err) + assert.Equal(t, "/api/v1/rerank", *path) +} + +func TestRejectsAnOutOfRangeIndex(t *testing.T) { + server, _, _ := serve(t, `{"results":[{"index":9,"relevance_score":0.5}]}`) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "out of range") +} + +func TestSurfacesTheVendorErrorBody(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(`{"error":{"message":"documents exceeds the limit"}}`)) + })) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "documents exceeds the limit") +} + +// truncate_prompt_tokens is a vLLM extension that appears in no managed +// vendor's documented schema. It is sent only where the vendor declared it +// and the operator opted in, and never by default — sending it unasked is how +// an undocumented field ends up on every request to every gateway. +func TestTruncatePromptTokensIsAbsentByDefault(t *testing.T) { + c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{}) + body, err := c.BuildRequestBody("q", []string{"d"}) + require.NoError(t, err) + assert.NotContains(t, body, "truncate_prompt_tokens") +} + +func TestTruncatePromptTokensIsSentWhenTheRowOptedIn(t *testing.T) { + c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{ + AcceptsTruncatePromptTokens: true, TruncatePromptTokens: 512, + }) + body, err := c.BuildRequestBody("q", []string{"d"}) + require.NoError(t, err) + assert.Equal(t, float64(512), body["truncate_prompt_tokens"]) +} diff --git a/internal/models/api/dashscoperank/client.go b/internal/models/api/dashscoperank/client.go new file mode 100644 index 00000000..1f1fc4c0 --- /dev/null +++ b/internal/models/api/dashscoperank/client.go @@ -0,0 +1,116 @@ +// Package dashscoperank implements Alibaba Model Studio's native text-rerank +// shape: the same fields as the Cohere dialect, wrapped in input/parameters +// and answered under output. +// +// https://help.aliyun.com/zh/model-studio/text-rerank-api +package dashscoperank + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" +) + +// Config is everything the client needs, already resolved by the catalog. +type Config struct { + Endpoint api.Endpoint + Settings catalog.RerankSettings +} + +// Client talks DashScope's native rerank to one endpoint. +type Client struct { + cfg Config +} + +// New builds a client. +func New(cfg Config) *Client { return &Client{cfg: cfg} } + +type request struct { + Model string `json:"model"` + Input input `json:"input"` + Parameters parameters `json:"parameters"` +} + +type input struct { + Query string `json:"query"` + Documents []string `json:"documents"` +} + +type parameters struct { + ReturnDocuments bool `json:"return_documents"` + TopN int `json:"top_n"` +} + +type response struct { + Output struct { + Results []struct { + Index int `json:"index"` + RelevanceScore float64 `json:"relevance_score"` + Document struct { + Text string `json:"text"` + } `json:"document"` + } `json:"results"` + } `json:"output"` + // DashScope reports failures in the body with a 200 on some paths. + Code string `json:"code"` + Message string `json:"message"` +} + +// BuildRequestBody is the golden-test entry point. +func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) { + body := request{ + Model: c.cfg.Endpoint.Model, + Input: input{Query: query, Documents: documents}, + Parameters: parameters{ + ReturnDocuments: c.cfg.Settings.SendReturnDocs, + // top_n is not optional in this shape: leaving it at zero would + // ask for no documents at all, so the full count is always sent. + TopN: len(documents), + }, + } + raw, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + for k, v := range c.cfg.Settings.ExtraBody { + if _, exists := out[k]; !exists { + out[k] = v + } + } + return out, nil +} + +// Rerank scores documents against the query. +func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) { + body, err := c.BuildRequestBody(query, documents) + if err != nil { + return nil, err + } + var decoded response + url := c.cfg.Endpoint.Resolve(c.cfg.Settings.Path) + if err := c.cfg.Endpoint.PostJSON(ctx, url, body, &decoded); err != nil { + return nil, err + } + if decoded.Code != "" { + return nil, fmt.Errorf("DashScope rerank error %s: %s", decoded.Code, decoded.Message) + } + out := make([]api.RerankResult, 0, len(decoded.Output.Results)) + for _, item := range decoded.Output.Results { + if item.Index < 0 || item.Index >= len(documents) { + return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents)) + } + out = append(out, api.RerankResult{ + Index: item.Index, + Score: item.RelevanceScore, + Text: item.Document.Text, + }) + } + return out, nil +} diff --git a/internal/models/api/dashscoperank/golden_test.go b/internal/models/api/dashscoperank/golden_test.go new file mode 100644 index 00000000..675cdca2 --- /dev/null +++ b/internal/models/api/dashscoperank/golden_test.go @@ -0,0 +1,116 @@ +package dashscoperank + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// documentedResponse is the response shape of DashScope's text-rerank API, +// copied from the reference so the decoder is pinned against the vendor's own +// document: https://help.aliyun.com/zh/model-studio/text-rerank-api +const documentedResponse = `{ + "output": { + "results": [ + {"document": {"text": "上海气候"}, "index": 1, "relevance_score": 0.7314}, + {"document": {"text": "北京美食"}, "index": 0, "relevance_score": 0.0002} + ] + }, + "usage": {"total_tokens": 24}, + "request_id": "4c5b1c4d-0e2f-9b1a-8a1e-1b2c3d4e5f60" +}` + +const endpointURL = "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank" + +func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client { + t.Helper() + return New(Config{ + Endpoint: api.Endpoint{BaseURL: url, Model: "gte-rerank-v2", Auth: api.BearerAuth("k")}, + Settings: settings, + }) +} + +func TestRequestBodyMatchesTheDocumentedSchema(t *testing.T) { + c := newClient(t, endpointURL, catalog.RerankSettings{SendReturnDocs: true}) + body, err := c.BuildRequestBody("上海天气", []string{"北京美食", "上海气候"}) + require.NoError(t, err) + + assert.Equal(t, map[string]any{ + "model": "gte-rerank-v2", + "input": map[string]any{ + "query": "上海天气", + "documents": []any{"北京美食", "上海气候"}, + }, + "parameters": map[string]any{ + "return_documents": true, + "top_n": float64(2), + }, + }, body) +} + +// top_n is mandatory in this shape: zero would ask for nothing back, so the +// full document count is always sent regardless of vendor settings. +func TestTopNAlwaysCarriesTheDocumentCount(t *testing.T) { + c := newClient(t, endpointURL, catalog.RerankSettings{}) + body, err := c.BuildRequestBody("q", []string{"a", "b", "c", "d"}) + require.NoError(t, err) + assert.Equal(t, float64(4), body["parameters"].(map[string]any)["top_n"]) +} + +func TestDecodesTheDocumentedResponse(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(documentedResponse)) + })) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + got, err := c.Rerank(context.Background(), "上海天气", []string{"北京美食", "上海气候"}) + require.NoError(t, err) + + assert.Equal(t, []api.RerankResult{ + {Index: 1, Score: 0.7314, Text: "上海气候"}, + {Index: 0, Score: 0.0002, Text: "北京美食"}, + }, got) +} + +// DashScope answers some failures with a 200 and an error code in the body, +// which would otherwise decode as an empty result set. +func TestSurfacesAnInBodyError(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"code":"InvalidParameter","message":"documents is too long","request_id":"r"}`)) + })) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "documents is too long") +} + +func TestPostsToTheBaseURLItself(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + var gotPath string + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"output":{"results":[]}}`)) + })) + defer server.Close() + + c := newClient(t, server.URL+"/api/v1/services/rerank/text-rerank/text-rerank", catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d"}) + require.NoError(t, err) + assert.Equal(t, "/api/v1/services/rerank/text-rerank/text-rerank", gotPath, + "the vendor's base URL already names the full endpoint") +} diff --git a/internal/models/api/endpoint.go b/internal/models/api/endpoint.go index 062a6480..e9f7ddd9 100644 --- a/internal/models/api/endpoint.go +++ b/internal/models/api/endpoint.go @@ -135,6 +135,40 @@ func (e Endpoint) Do(req *http.Request) (*http.Response, error) { return resp, nil } +// PostJSON sends one authenticated JSON request and decodes the reply into +// out. It is the whole round-trip for the non-streaming protocols (rerank, +// embeddings), which have no SSE to assemble. +// +// It never returns a nil error with nothing decoded: every failure path — +// marshalling, SSRF, transport, non-2xx, decoding — returns an error, so a +// caller cannot mistake an empty result for a successful empty answer. +func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error { + req, _, err := e.NewRequest(ctx, url, body, false) + if err != nil { + return err + } + // No LogRequest here: the chat protocols log their own bodies because a + // prompt is what an operator debugs, while a rerank body is a query plus + // every candidate chunk. The rerank layer logs a truncated line at Debug + // instead, so this would have duplicated it at Info. + resp, err := e.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read response: %w", err) + } + if out == nil { + return nil + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("decode response: %w", err) + } + return nil +} + // HTTPError is a non-2xx vendor reply. type HTTPError struct { StatusCode int diff --git a/internal/models/api/nimrerank/client.go b/internal/models/api/nimrerank/client.go new file mode 100644 index 00000000..4eaefbef --- /dev/null +++ b/internal/models/api/nimrerank/client.go @@ -0,0 +1,106 @@ +// Package nimrerank implements NVIDIA NIM's retrieval reranking shape: a +// query object, a passages array, and rankings carrying a logit. +// +// Two things separate it from the Cohere dialect and both matter to callers. +// The score is an unbounded log-odds value that is routinely negative, not a +// 0..1 relevance — catalog.RerankSettings.ScoreScale says so. And `truncate` +// defaults to "NONE", which means an over-long passage fails the request +// instead of being cut, so the vendor sets it explicitly. +// +// https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer +package nimrerank + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" +) + +// Config is everything the client needs, already resolved by the catalog. +type Config struct { + Endpoint api.Endpoint + Settings catalog.RerankSettings +} + +// Client talks NIM reranking to one endpoint. +type Client struct { + cfg Config +} + +// New builds a client. +func New(cfg Config) *Client { return &Client{cfg: cfg} } + +type text struct { + Text string `json:"text"` +} + +type request struct { + Model string `json:"model"` + Query text `json:"query"` + Passages []text `json:"passages"` + Truncate string `json:"truncate,omitempty"` +} + +type response struct { + Rankings []struct { + Index int `json:"index"` + Logit float64 `json:"logit"` + } `json:"rankings"` +} + +// BuildRequestBody is the golden-test entry point. +func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) { + passages := make([]text, len(documents)) + for i := range documents { + passages[i] = text{Text: documents[i]} + } + body := request{ + Model: c.cfg.Endpoint.Model, + Query: text{Text: query}, + Passages: passages, + Truncate: c.cfg.Settings.Truncate, + } + raw, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + for k, v := range c.cfg.Settings.ExtraBody { + if _, exists := out[k]; !exists { + out[k] = v + } + } + return out, nil +} + +// Rerank scores documents against the query. The returned scores are logits; +// see the package comment. +func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) { + body, err := c.BuildRequestBody(query, documents) + if err != nil { + return nil, err + } + var decoded response + url := c.cfg.Endpoint.Resolve(c.cfg.Settings.Path) + if err := c.cfg.Endpoint.PostJSON(ctx, url, body, &decoded); err != nil { + return nil, err + } + out := make([]api.RerankResult, 0, len(decoded.Rankings)) + for _, item := range decoded.Rankings { + if item.Index < 0 || item.Index >= len(documents) { + return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents)) + } + out = append(out, api.RerankResult{ + Index: item.Index, + Score: item.Logit, + Text: documents[item.Index], + }) + } + return out, nil +} diff --git a/internal/models/api/nimrerank/golden_test.go b/internal/models/api/nimrerank/golden_test.go new file mode 100644 index 00000000..daa93f29 --- /dev/null +++ b/internal/models/api/nimrerank/golden_test.go @@ -0,0 +1,128 @@ +package nimrerank + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// documentedResponse is the response example from NVIDIA's reranking +// reference, copied verbatim so the decoder is pinned against the vendor's +// own document rather than against our structs: +// https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer +// +// Note the negative logit. This protocol does not return a 0..1 relevance +// score, which is why catalog.RerankSettings.ScoreScale exists. +const documentedResponse = `{ + "rankings": [ + {"index": 2, "logit": 0.226318359375}, + {"index": 1, "logit": -1.171875}, + {"index": 0, "logit": -6.3125} + ] +}` + +func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client { + t.Helper() + return New(Config{ + Endpoint: api.Endpoint{BaseURL: url, Model: "nvidia/nv-rerankqa-mistral-4b-v3", Auth: api.BearerAuth("k")}, + Settings: settings, + }) +} + +func TestRequestBodyMatchesTheDocumentedSchema(t *testing.T) { + c := newClient(t, "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", + catalog.RerankSettings{Truncate: "END", ScoreScale: api.ScoreLogit}) + + body, err := c.BuildRequestBody("q", []string{"d0", "d1"}) + require.NoError(t, err) + + assert.Equal(t, map[string]any{ + "model": "nvidia/nv-rerankqa-mistral-4b-v3", + "query": map[string]any{"text": "q"}, + "passages": []any{map[string]any{"text": "d0"}, map[string]any{"text": "d1"}}, + "truncate": "END", + }, body) +} + +// truncate defaults to NONE upstream, which fails the request on an over-long +// passage instead of cutting it. A vendor that does not set it must not have +// the key invented for it either. +func TestTruncateIsOmittedWhenUnset(t *testing.T) { + c := newClient(t, "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", catalog.RerankSettings{}) + body, err := c.BuildRequestBody("q", []string{"d0"}) + require.NoError(t, err) + assert.NotContains(t, body, "truncate") +} + +func TestDecodesTheDocumentedResponse(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(documentedResponse)) + })) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{ScoreScale: api.ScoreLogit}) + got, err := c.Rerank(context.Background(), "q", []string{"d0", "d1", "d2"}) + require.NoError(t, err) + + assert.Equal(t, []api.RerankResult{ + {Index: 2, Score: 0.226318359375, Text: "d2"}, + {Index: 1, Score: -1.171875, Text: "d1"}, + {Index: 0, Score: -6.3125, Text: "d0"}, + }, got) +} + +// An index the request never sent is a protocol violation, not something to +// map onto whatever document happens to sit at that offset. +func TestRejectsAnOutOfRangeIndex(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"rankings":[{"index":7,"logit":0.5}]}`)) + })) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d0"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "out of range") +} + +func TestSurfacesTheVendorErrorBody(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"detail":"input too long"}`)) + })) + defer server.Close() + + c := newClient(t, server.URL, catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d0"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "input too long") +} + +func TestRequestReachesTheConfiguredURL(t *testing.T) { + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + var gotPath string + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"rankings":[]}`)) + })) + defer server.Close() + + c := newClient(t, server.URL+"/v1/retrieval/nvidia/reranking", catalog.RerankSettings{}) + _, err := c.Rerank(context.Background(), "q", []string{"d0"}) + require.NoError(t, err) + assert.Equal(t, "/v1/retrieval/nvidia/reranking", gotPath, "the base URL already names the endpoint") + assert.Equal(t, "q", gotBody["query"].(map[string]any)["text"]) +} diff --git a/internal/models/api/rerank.go b/internal/models/api/rerank.go new file mode 100644 index 00000000..e2465a05 --- /dev/null +++ b/internal/models/api/rerank.go @@ -0,0 +1,83 @@ +package api + +import "context" + +// RerankAPI names a rerank wire protocol. It is deliberately a separate type +// from API, the chat protocol selector: the two vocabularies must not mix, or +// a chat row whose extra_config.api names a rerank dialect would pass +// validation and fail at call time. +type RerankAPI string + +// The rerank protocols WeKnora speaks. +const ( + // RerankCohere is the de-facto standard shape — POST {base}/rerank with + // {model, query, documents} and a results array of {index, + // relevance_score, document}. Cohere defined it; Jina, Zhipu, + // SiliconFlow, Qianfan, GPUStack and every OpenAI-compatible gateway + // that serves rerank at all copy it. + RerankCohere RerankAPI = "cohere-rerank" + // RerankDashScope is Alibaba Model Studio's native shape, which wraps + // the same fields in input/parameters and answers under output. + RerankDashScope RerankAPI = "dashscope-rerank" + // RerankNIM is NVIDIA NIM's retrieval shape: a query object, a passages + // array, and rankings carrying an unbounded logit instead of a + // probability. + RerankNIM RerankAPI = "nim-rerank" + // RerankTencentLKEAP is Tencent Cloud's RunRerank action, reached through + // the official SDK because it is TC3-signed rather than key-authenticated. + RerankTencentLKEAP RerankAPI = "tencent-lkeap" + // RerankVolcengineKnowledge is Volcengine's managed Knowledge Service + // rerank, reached through the vikingdb SDK for the same reason. + RerankVolcengineKnowledge RerankAPI = "volcengine-knowledge" +) + +// Known reports whether the value names a protocol this build implements. +func (a RerankAPI) Known() bool { + switch a { + case RerankCohere, RerankDashScope, RerankNIM, + RerankTencentLKEAP, RerankVolcengineKnowledge: + return true + } + return false +} + +// HTTPServed reports whether a protocol package implements the protocol. The +// SDK-backed ones are built by the rerank package itself, which is where the +// vendor SDKs are imported. +func (a RerankAPI) HTTPServed() bool { + switch a { + case RerankCohere, RerankDashScope, RerankNIM: + return true + } + return false +} + +// ScoreScale describes what a rerank score means, which differs per protocol +// and cannot be inferred from the number itself. +type ScoreScale string + +const ( + // ScoreProbability is a 0..1 relevance score. Every protocol but NIM + // returns one. + ScoreProbability ScoreScale = "probability" + // ScoreLogit is an unbounded log-odds value that may be negative. NIM + // returns one, so a threshold tuned for probabilities rejects almost + // everything on that vendor. + ScoreLogit ScoreScale = "logit" +) + +// RerankResult is one scored document, in the vendor's own score scale. +// Index refers to the position in the documents slice that was sent. +type RerankResult struct { + Index int + Score float64 + // Text is the document echoed back, empty when the vendor was not asked + // to return documents. Callers index into their own slice instead of + // relying on it. + Text string +} + +// Reranker is what every rerank protocol package implements. +type Reranker interface { + Rerank(ctx context.Context, query string, documents []string) ([]RerankResult, error) +} diff --git a/internal/models/catalog/compat.go b/internal/models/catalog/compat.go index 0fb7b69b..d6956655 100644 --- a/internal/models/catalog/compat.go +++ b/internal/models/catalog/compat.go @@ -341,6 +341,9 @@ type VendorCompat struct { OpenAIResponses OpenAIResponsesCompat `json:"openai_responses,omitempty"` AnthropicMessages AnthropicMessagesCompat `json:"anthropic_messages,omitempty"` GoogleGenerativeAI GoogleGenerativeAICompat `json:"google_generative_ai,omitempty"` + // Rerank is the rerank protocol overlay. It has no per-protocol variants: + // a vendor serves exactly one rerank dialect. + Rerank RerankCompat `json:"rerank,omitempty"` } // apply writes the set fields of an overlay struct onto a settings struct diff --git a/internal/models/catalog/legacy_rows_test.go b/internal/models/catalog/legacy_rows_test.go index 8817a020..14a0a4f1 100644 --- a/internal/models/catalog/legacy_rows_test.go +++ b/internal/models/catalog/legacy_rows_test.go @@ -194,7 +194,14 @@ func TestLegacyRowsValidateAndResolve(t *testing.T) { if resolved.Vendor == nil { t.Fatal("resolved with a nil vendor") } - if !resolved.API.Known() { + // Rerank rows resolve to a rerank protocol; everything else + // resolves to a chat one. The two vocabularies are separate + // types so a row can never land on the wrong one. + if row.typ == types.ModelTypeRerank { + if !resolved.RerankAPI.Known() { + t.Fatalf("resolved to an unknown rerank protocol %q", resolved.RerankAPI) + } + } else if !resolved.API.Known() { t.Fatalf("resolved to an unknown protocol %q", resolved.API) } if resolved.RemoteModel == "" && row.model != "" { diff --git a/internal/models/catalog/registry.go b/internal/models/catalog/registry.go index 009822ff..b073762e 100644 --- a/internal/models/catalog/registry.go +++ b/internal/models/catalog/registry.go @@ -31,6 +31,9 @@ func Register(v *Vendor) { if v.Auth == "" { v.Auth = AuthBearer } + if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) { + v.RerankAPI = api.RerankCohere + } for i := range v.Models { if v.Models[i].Type == "" { v.Models[i].Type = types.ModelTypeKnowledgeQA diff --git a/internal/models/catalog/rerank.go b/internal/models/catalog/rerank.go new file mode 100644 index 00000000..dec19f6d --- /dev/null +++ b/internal/models/catalog/rerank.go @@ -0,0 +1,92 @@ +package catalog + +import "github.com/Tencent/WeKnora/internal/models/api" + +// RerankCompat is the overlay form (every field optional) of the rerank +// settings. JSON keys are the documented names used in models.json and +// config/models.json. +type RerankCompat struct { + Path *string `json:"path,omitempty"` + SendTopN *bool `json:"send_top_n,omitempty"` + SendReturnDocs *bool `json:"send_return_documents,omitempty"` + ScoreScale *api.ScoreScale `json:"score_scale,omitempty"` + Truncate *string `json:"truncate,omitempty"` + MaxDocuments *int `json:"max_documents,omitempty"` + MaxQueryChars *int `json:"max_query_chars,omitempty"` + MaxDocumentChars *int `json:"max_document_chars,omitempty"` + MaxRequestChars *int `json:"max_request_chars,omitempty"` + MaxConcurrency *int `json:"max_concurrency,omitempty"` + RequestTimeout *int `json:"request_timeout_seconds,omitempty"` + // AcceptsTruncatePromptTokens marks a vLLM-class runtime. + AcceptsTruncatePromptTokens *bool `json:"accepts_truncate_prompt_tokens,omitempty"` + // UnsupportedReason marks a model this build cannot call. + UnsupportedReason *string `json:"unsupported_reason,omitempty"` + ExtraBody map[string]any `json:"extra_body,omitempty"` +} + +// RerankSettings is the resolved (fully defaulted) form. +type RerankSettings struct { + // Path is appended to the base URL by protocols that use one. Vendors + // whose default base URL already names the full endpoint leave it empty. + Path string + // SendTopN asks for every document back by naming their count. Omitting + // it means "all", which is what every protocol here does by default. + SendTopN bool + // SendReturnDocs asks the vendor to echo the document text. Callers index + // into their own slice, so this is off unless a vendor needs it. + SendReturnDocs bool + // ScoreScale says what the returned numbers mean. See api.ScoreScale: + // getting this wrong makes a relevance threshold meaningless. + ScoreScale api.ScoreScale + // Truncate is the vendor's server-side truncation setting for + // over-long input ("END" on NIM). Empty sends nothing. + Truncate string + // Limits carry the documented per-request ceilings. MaxQueryChars is + // checked on its own rather than through BatchLimits: the query is not an + // item to be split, so an over-long one cannot be made to fit by batching. + MaxDocuments int + MaxQueryChars int + MaxDocumentChars int + MaxRequestChars int + // MaxConcurrency bounds in-flight batches when the input is split. 0 + // means the caller's default. + MaxConcurrency int + // RequestTimeout caps one request, in seconds. 0 leaves the HTTP client + // without its own deadline and lets the caller's context govern, which is + // what every vendor but WeKnora Cloud has always done here. + RequestTimeout int + // AcceptsTruncatePromptTokens reports that this vendor is a vLLM-class + // runtime, which is the only kind that implements the + // `truncate_prompt_tokens` extension. It is not part of the Cohere shape + // and appears in no managed vendor's schema, so it must never be sent to + // one: that is how an undocumented field ends up on every request. + AcceptsTruncatePromptTokens bool + // UnsupportedReason is set on a catalog entry naming a rerank dialect no + // protocol package implements. Resolve refuses it rather than letting a + // request go out shaped for a different protocol, which surfaces as a + // decode error far from its cause. + UnsupportedReason string + // TruncatePromptTokens is the budget the operator opted into on this row, + // honoured only where AcceptsTruncatePromptTokens is set. It is a row + // setting rather than a vendor fact because one runtime serves models + // with different context windows. Zero sends nothing. + TruncatePromptTokens int + ExtraBody map[string]any +} + +// BatchLimits renders the documented ceilings for api.SplitBatches. +func (s RerankSettings) BatchLimits() api.BatchLimits { + return api.BatchLimits{ + MaxItems: s.MaxDocuments, + MaxItemRunes: s.MaxDocumentChars, + MaxTotalRunes: s.MaxRequestChars, + } +} + +// DefaultRerank is the protocol baseline: ask for every document, expect a +// 0..1 relevance score, enforce no ceiling the vendor did not state. +func DefaultRerank() RerankSettings { + return RerankSettings{ + ScoreScale: api.ScoreProbability, + } +} diff --git a/internal/models/catalog/resolve.go b/internal/models/catalog/resolve.go index dcf445c1..bae3227b 100644 --- a/internal/models/catalog/resolve.go +++ b/internal/models/catalog/resolve.go @@ -3,6 +3,7 @@ package catalog import ( "encoding/json" "fmt" + "strconv" "strings" "github.com/Tencent/WeKnora/internal/models/api" @@ -21,6 +22,16 @@ const ( // ExtraThinkingControl is the legacy thinking encoding selector written // by older UIs: none | enable_thinking | thinking_type | chat_template_kwargs. ExtraThinkingControl = "thinking_control" + // ExtraTruncatePromptTokens is the vLLM-only server-side truncation + // budget for rerank, opt-in per row. It is never sent unless the operator + // set it: vendors that do not implement it reject the unknown field. + ExtraTruncatePromptTokens = "truncate_prompt_tokens" + // ExtraScoreScale overrides the vendor rerank score scale for one row. + // A self-hosted gateway serves whatever reranker was deployed behind it + // and the two families disagree: BGE-class models answer a 0..1 + // probability, Qwen3-Reranker-class models an unbounded score. A vendor + // can only state what its own documentation shows. + ExtraScoreScale = "score_scale" ) // Ref identifies one configured model. @@ -51,6 +62,12 @@ type Resolved struct { OpenAIResponses OpenAIResponsesSettings AnthropicMessages AnthropicMessagesSettings GoogleGenerativeAI GoogleGenerativeAISettings + + // RerankAPI and Rerank are filled only when the reference asked for a + // rerank model. Chat resolution is on every request's hot path, so the + // rerank overlay is not merged for it. + RerankAPI api.RerankAPI + Rerank RerankSettings } // Resolve merges the vendor, catalog entry, extra-config and per-row @@ -86,6 +103,10 @@ func Resolve(ref Ref) (*Resolved, error) { applySpecOverride(&spec, ref.Override) } + if modelType == types.ModelTypeRerank { + return resolveRerank(ref, vendor, spec, cataloged, baseURL) + } + resolvedAPI := spec.API if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" { resolvedAPI = api.API(forced) @@ -346,3 +367,84 @@ func (r *Resolved) Capabilities() Capabilities { caps.ThinkingLevels = r.ThinkingLevels.SupportedLevels() return caps } + +// resolveRerank merges the rerank layers. Rerank has one settings struct +// rather than one per protocol, so the model entry's compat object is decoded +// unconditionally instead of being matched against a protocol. +func resolveRerank( + ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string, +) (*Resolved, error) { + protocol := vendor.RerankAPI + if protocol == "" { + protocol = api.RerankCohere + } + if !protocol.Known() { + return nil, fmt.Errorf("catalog: unknown rerank api %q for provider %s", protocol, vendor.ID) + } + + // Lowest precedence first: protocol default, vendor, model entry, row spec + // override. extra_config is the operator speaking about this one row, so it + // comes last. + settings := DefaultRerank() + apply(&settings, &vendor.Compat.Rerank) + if len(spec.Compat) > 0 { + overlay := &RerankCompat{} + if err := decodeCompat(spec.Compat, overlay); err != nil { + return nil, fmt.Errorf("rerank compat: %w", err) + } + apply(&settings, overlay) + } + if raw := ref.Override.CompatJSON(); len(raw) > 0 { + overlay := &RerankCompat{} + if err := decodeCompat(raw, overlay); err != nil { + return nil, fmt.Errorf("rerank compat: %w", err) + } + apply(&settings, overlay) + } + + if raw := strings.TrimSpace(ref.Extra[ExtraScoreScale]); raw != "" { + scale := api.ScoreScale(strings.ToLower(raw)) + if scale != api.ScoreProbability && scale != api.ScoreLogit { + return nil, fmt.Errorf( + "catalog: invalid %s in extra_config: %q (expected %q or %q)", + ExtraScoreScale, raw, api.ScoreProbability, api.ScoreLogit, + ) + } + settings.ScoreScale = scale + } + if raw := strings.TrimSpace(ref.Extra[ExtraTruncatePromptTokens]); raw != "" { + if !settings.AcceptsTruncatePromptTokens { + return nil, fmt.Errorf( + "catalog: %s is a vLLM extension and %s does not implement it; "+ + "remove it from extra_config (it is accepted by self-hosted runtimes only)", + ExtraTruncatePromptTokens, vendor.ID, + ) + } + budget, err := strconv.Atoi(raw) + if err != nil || budget <= 0 { + return nil, fmt.Errorf("catalog: invalid %s in extra_config: %q", ExtraTruncatePromptTokens, raw) + } + settings.TruncatePromptTokens = budget + } + // Checked after every layer: a model entry is what declares a dialect + // this build cannot speak. + if settings.UnsupportedReason != "" { + return nil, fmt.Errorf( + "catalog: %s does not serve %q through a protocol this build implements: %s", + vendor.ID, spec.ID, settings.UnsupportedReason, + ) + } + out := &Resolved{ + Vendor: vendor, + Spec: spec, + Cataloged: cataloged, + BaseURL: baseURL, + RemoteModel: ref.Model, + RerankAPI: protocol, + Rerank: settings, + } + if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" { + out.RemoteModel = override + } + return out, nil +} diff --git a/internal/models/catalog/types.go b/internal/models/catalog/types.go index f3421cec..24e217d2 100644 --- a/internal/models/catalog/types.go +++ b/internal/models/catalog/types.go @@ -42,14 +42,16 @@ const ( // ExtraField describes one vendor-specific configuration input the model // editor renders dynamically (Azure api-version, LKEAP secret key, ...). type ExtraField struct { - Key string `json:"key"` - Label string `json:"label"` - Labels map[string]string `json:"labels,omitempty"` - Type string `json:"type"` // "string", "number", "boolean", "select", "password" - Required bool `json:"required"` - Default string `json:"default,omitempty"` - Placeholder string `json:"placeholder,omitempty"` - Options []ExtraFieldOption `json:"options,omitempty"` + Key string `json:"key"` + Label string `json:"label"` + Labels map[string]string `json:"labels,omitempty"` + Type string `json:"type"` // "string", "number", "boolean", "select", "password" + Required bool `json:"required"` + Default string `json:"default,omitempty"` + Placeholder string `json:"placeholder,omitempty"` + // Placeholders carries localized variants of Placeholder. + Placeholders map[string]string `json:"placeholders,omitempty"` + Options []ExtraFieldOption `json:"options,omitempty"` // ModelTypes restricts the field to some model types; empty means all. ModelTypes []types.ModelType `json:"model_types,omitempty"` // Secret marks values that must never be echoed back to the UI. @@ -102,6 +104,11 @@ func localizedOr(table map[string]string, locale, fallback string) string { return fallback } +// LocalizedPlaceholder resolves the placeholder for a locale. +func (f ExtraField) LocalizedPlaceholder(locale string) string { + return localizedOr(f.Placeholders, locale, f.Placeholder) +} + // CredentialLabelFor returns the credential naming for a model type, or nil // when the vendor uses the generic API-key wording. func (v *Vendor) CredentialLabelFor(modelType types.ModelType) *CredentialLabel { @@ -119,10 +126,19 @@ func (v *Vendor) CredentialLabelFor(modelType types.ModelType) *CredentialLabel return nil } -// ExtraFieldOption is one choice of a select field. +// ExtraFieldOption is one choice of a select field. Labels carries localized +// variants keyed by locale, like every other operator-facing string here: an +// option whose label is prose rather than an identifier is unreadable to half +// the product without it. type ExtraFieldOption struct { - Label string `json:"label"` - Value string `json:"value"` + Label string `json:"label"` + Labels map[string]string `json:"labels,omitempty"` + Value string `json:"value"` +} + +// LocalizedLabel resolves the option label for a locale. +func (o ExtraFieldOption) LocalizedLabel(locale string) string { + return localizedOr(o.Labels, locale, o.Label) } // ModelCost is priced per million tokens, in USD unless Currency says otherwise. @@ -223,6 +239,9 @@ type Vendor struct { Icon []byte // API is the default chat protocol; a ModelSpec may override it. API api.API + // RerankAPI is the rerank protocol. Register defaults it to the Cohere + // shape for any vendor that serves rerank without naming another. + RerankAPI api.RerankAPI // DefaultBaseURLs by model type; GetDefaultURL falls back to chat. DefaultBaseURLs map[types.ModelType]string ModelTypes []types.ModelType diff --git a/internal/models/parity/parity_test.go b/internal/models/parity/parity_test.go index fc1f7420..9b10629e 100644 --- a/internal/models/parity/parity_test.go +++ b/internal/models/parity/parity_test.go @@ -16,6 +16,7 @@ package parity import ( + "bytes" "context" "encoding/json" "net/http" @@ -523,6 +524,13 @@ func TestCatalogInvariants(t *testing.T) { // Resolve validates the compat object against the model's // protocol: an unknown or misspelled key fails here. _, err := catalog.Resolve(catalog.Ref{Provider: v.ID, Model: m.ID, ModelType: m.Type}) + if bytes.Contains(m.Compat, []byte("unsupported_reason")) { + // An entry that declares itself unserveable must say so + // by refusing, not by resolving into a request shaped + // for a protocol this build does not speak. + assert.Error(t, err, "%s: declares unsupported_reason but still resolves", label) + continue + } assert.NoError(t, err, "%s: does not resolve", label) } } diff --git a/internal/models/parity/rerank_test.go b/internal/models/parity/rerank_test.go new file mode 100644 index 00000000..b717437e --- /dev/null +++ b/internal/models/parity/rerank_test.go @@ -0,0 +1,522 @@ +package parity + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestEveryRerankVendorResolvesToAKnownProtocol walks the catalog, so a newly +// added vendor is covered without touching this file. +func TestEveryRerankVendorResolvesToAKnownProtocol(t *testing.T) { + vendors := catalog.ListByType(types.ModelTypeRerank) + require.NotEmpty(t, vendors, "no vendor serves rerank") + + for _, v := range vendors { + t.Run(v.ID, func(t *testing.T) { + model := "some-rerank-model" + if catalogued := v.ModelsByType(types.ModelTypeRerank); len(catalogued) > 0 { + model = catalogued[0].ID + } + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: v.ID, Model: model, ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + assert.True(t, resolved.RerankAPI.Known(), + "unknown rerank protocol %q", resolved.RerankAPI) + // The catch-all vendor has no endpoint of its own: the operator + // supplies one, and Validate requires it. + if v.ID != catalog.GenericID { + assert.NotEmpty(t, resolved.BaseURL, "rerank vendors need a default endpoint") + } + + // A vendor that declares a ceiling must declare a usable one. + assert.GreaterOrEqual(t, resolved.Rerank.MaxDocuments, 0) + assert.GreaterOrEqual(t, resolved.Rerank.MaxDocumentChars, 0) + if resolved.Rerank.ScoreScale != "" { + assert.Contains(t, + []api.ScoreScale{api.ScoreProbability, api.ScoreLogit}, + resolved.Rerank.ScoreScale) + } + }) + } +} + +// TestRerankProtocolAssignment pins which dialect each vendor speaks. The +// wire shapes are mutually incompatible, so a silent reassignment breaks +// every rerank call for that vendor. +func TestRerankProtocolAssignment(t *testing.T) { + for id, want := range map[string]api.RerankAPI{ + "aliyun": api.RerankDashScope, + "nvidia": api.RerankNIM, + "lkeap": api.RerankTencentLKEAP, + "volcengine": api.RerankVolcengineKnowledge, + "zhipu": api.RerankCohere, + "jina": api.RerankCohere, + "siliconflow": api.RerankCohere, + "qianfan": api.RerankCohere, + "gpustack": api.RerankCohere, + "generic": api.RerankCohere, + "openai": api.RerankCohere, + "weknoracloud": api.RerankCohere, + "novita": api.RerankCohere, + "openrouter": api.RerankCohere, + "litellm": api.RerankCohere, + } { + t.Run(id, func(t *testing.T) { + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: id, Model: "m", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + assert.Equal(t, want, resolved.RerankAPI) + }) + } +} + +// TestRerankOutboundShapePerProtocol builds the real client through +// rerank.NewReranker and asserts what each dialect puts on the wire. +func TestRerankOutboundShapePerProtocol(t *testing.T) { + for _, tc := range []struct { + name string + provider string + model string + assert func(t *testing.T, path string, body map[string]any) + reply string + }{ + { + name: "cohere shape", provider: "siliconflow", model: "BAAI/bge-reranker-v2-m3", + reply: `{"results":[{"index":0,"relevance_score":0.5}]}`, + assert: func(t *testing.T, path string, body map[string]any) { + assert.Equal(t, "/v1/rerank", path) + assert.Equal(t, "BAAI/bge-reranker-v2-m3", body["model"]) + assert.Equal(t, "q", body["query"]) + assert.Equal(t, []any{"d0"}, body["documents"]) + }, + }, + { + name: "dashscope shape", provider: "aliyun", model: "gte-rerank-v2", + reply: `{"output":{"results":[{"index":0,"relevance_score":0.5,"document":{"text":"d0"}}]}}`, + assert: func(t *testing.T, _ string, body map[string]any) { + input, ok := body["input"].(map[string]any) + require.True(t, ok, "DashScope wraps the query and documents in input") + assert.Equal(t, "q", input["query"]) + assert.Contains(t, body, "parameters") + assert.NotContains(t, body, "documents", "documents live under input") + }, + }, + { + name: "nim shape", provider: "nvidia", model: "nvidia/nv-rerankqa-mistral-4b-v3", + reply: `{"rankings":[{"index":0,"logit":0.0}]}`, + assert: func(t *testing.T, _ string, body map[string]any) { + query, ok := body["query"].(map[string]any) + require.True(t, ok, "NIM sends the query as an object") + assert.Equal(t, "q", query["text"]) + assert.Equal(t, []any{map[string]any{"text": "d0"}}, body["passages"]) + assert.Equal(t, "END", body["truncate"], "NIM fails over-long input unless told to cut") + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var gotPath string + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(tc.reply)) + })) + defer server.Close() + + reranker, err := rerank.NewReranker(&rerank.RerankerConfig{ + Provider: tc.provider, ModelName: tc.model, APIKey: "k", + BaseURL: server.URL + rerankPathFor(t, tc.provider), + }) + require.NoError(t, err) + + _, err = reranker.Rerank(context.Background(), "q", []string{"d0"}) + require.NoError(t, err) + tc.assert(t, gotPath, gotBody) + }) + } +} + +// rerankPathFor keeps the loopback base URL shaped like the vendor's own, so +// the test exercises the same path handling production does. +func rerankPathFor(t *testing.T, provider string) string { + t.Helper() + switch provider { + case "aliyun": + return "/api/v1/services/rerank/text-rerank/text-rerank" + case "nvidia": + return "/v1/retrieval/nvidia/reranking" + default: + return "/v1" + } +} + +// TestNvidiaRerankScoresAreComparable is the end-to-end form of the unit test +// in the rerank package: a NIM logit must reach the retrieval pipeline on the +// same 0..1 scale as every other vendor, because they all meet one +// RerankThreshold. +func TestNvidiaRerankScoresAreComparable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"rankings":[{"index":0,"logit":-1.171875},{"index":1,"logit":4.0}]}`)) + })) + defer server.Close() + + reranker, err := rerank.NewReranker(&rerank.RerankerConfig{ + Provider: "nvidia", ModelName: "nvidia/nv-rerankqa-mistral-4b-v3", APIKey: "k", + BaseURL: server.URL + "/v1/retrieval/nvidia/reranking", + }) + require.NoError(t, err) + + got, err := reranker.Rerank(context.Background(), "q", []string{"d0", "d1"}) + require.NoError(t, err) + require.Len(t, got, 2) + for _, item := range got { + assert.GreaterOrEqual(t, item.RelevanceScore, 0.0) + assert.LessOrEqual(t, item.RelevanceScore, 1.0) + } + // Results come back ranked, so the logit of 4.0 leads and the negative + // one follows: the conversion is monotonic and the ranking is preserved. + assert.Equal(t, 1, got[0].Index) + assert.Greater(t, got[0].RelevanceScore, got[1].RelevanceScore) +} + +// TestSignedRerankVendorRequiresItsIdentityPair keeps the pre-catalog +// contract: WeKnora Cloud rerank is signed with an AppID / AppSecret pair, and +// a row without one must fail at construction rather than send unsigned +// requests that the far end rejects with something less readable. +func TestSignedRerankVendorRequiresItsIdentityPair(t *testing.T) { + _, err := rerank.NewReranker(&rerank.RerankerConfig{ + Provider: "weknoracloud", ModelName: "rerank", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "AppID") + + _, err = rerank.NewReranker(&rerank.RerankerConfig{ + Provider: "weknoracloud", ModelName: "rerank", AppID: "app", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "AppSecret") +} + +// TestTruncatePromptTokensOnlyReachesVLLMClassVendors pins the gate on the +// vLLM extension restored from the pre-catalog client. It is opt-in per row, +// but the row can only opt into it on a runtime that implements it: no +// managed vendor documents the field, and sending it to one is how an +// undocumented parameter ends up on every request. +func TestTruncatePromptTokensOnlyReachesVLLMClassVendors(t *testing.T) { + optIn := map[string]string{catalog.ExtraTruncatePromptTokens: "512"} + + for _, id := range []string{"generic", "gpustack"} { + t.Run(id+" accepts it", func(t *testing.T) { + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: id, Model: "m", ModelType: types.ModelTypeRerank, + BaseURL: "http://127.0.0.1:9/v1", Extra: optIn, + }) + require.NoError(t, err) + assert.Equal(t, 512, resolved.Rerank.TruncatePromptTokens) + }) + } + + for _, id := range []string{"jina", "zhipu", "siliconflow", "qianfan", "weknoracloud", "aliyun", "nvidia"} { + t.Run(id+" rejects it", func(t *testing.T) { + _, err := catalog.Resolve(catalog.Ref{ + Provider: id, Model: "m", ModelType: types.ModelTypeRerank, Extra: optIn, + }) + require.Error(t, err, "a managed vendor must not silently accept a vLLM extension") + assert.Contains(t, err.Error(), "vLLM extension") + }) + } +} + +func TestTruncatePromptTokensRejectsAnInvalidValue(t *testing.T) { + for _, raw := range []string{"0", "-1", "abc"} { + _, err := catalog.Resolve(catalog.Ref{ + Provider: "generic", Model: "m", ModelType: types.ModelTypeRerank, + BaseURL: "http://127.0.0.1:9/v1", + Extra: map[string]string{catalog.ExtraTruncatePromptTokens: raw}, + }) + require.Error(t, err, "value %q", raw) + } +} + +// TestRerankCeilingsAreTheDocumentedOnes pins the numbers the shared batching +// layer enforces. They used to live inside each client, where a dedicated +// test watched them; now they are declarations, so they need their own guard — +// TestEveryRerankVendorResolvesToAKnownProtocol only checks they are sane. +func TestRerankCeilingsAreTheDocumentedOnes(t *testing.T) { + for id, want := range map[string]catalog.RerankSettings{ + // docs.bigmodel.cn: 最多 128 条,query 与单条文档各 4096 字符 + "zhipu": {MaxDocuments: 128, MaxQueryChars: 4096, MaxDocumentChars: 4096}, + // cloud.tencent.com/document/product/1772: RunRerank 60 docs, + // Query + Docs together 2000 characters, one request at a time. + "lkeap": {MaxDocuments: 60, MaxRequestChars: 2000, MaxConcurrency: 1}, + // Ark Knowledge Service rerank: 50 per request. + "volcengine": {MaxDocuments: 50, MaxConcurrency: 4}, + // NIM reranking: passages is capped at 512 items. + "nvidia": {MaxDocuments: 512}, + // cloud.baidu.com/doc/qianfan-api: 文本数量不超过64, query 不超过 + // 1600 个字符, 每条 document 不超过 4096 个字符. + "qianfan": {MaxDocuments: 64, MaxQueryChars: 1600, MaxDocumentChars: 4096}, + // help.aliyun.com text-rerank: 500 documents per request. Its length + // limits are stated in tokens, which runes cannot express. + "aliyun": {MaxDocuments: 500}, + } { + t.Run(id, func(t *testing.T) { + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: id, Model: "m", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + got := resolved.Rerank + assert.Equal(t, want.MaxDocuments, got.MaxDocuments, "max documents") + assert.Equal(t, want.MaxQueryChars, got.MaxQueryChars, "max query characters") + assert.Equal(t, want.MaxDocumentChars, got.MaxDocumentChars, "max document characters") + assert.Equal(t, want.MaxRequestChars, got.MaxRequestChars, "max request characters") + assert.Equal(t, want.MaxConcurrency, got.MaxConcurrency, "max concurrency") + }) + } +} + +// TestWeKnoraCloudRerankKeepsItsDeadline pins the one rerank vendor that has +// ever had a client-level timeout. Without it a hung endpoint holds the +// retrieval stage open for as long as the caller's context allows. +func TestWeKnoraCloudRerankKeepsItsDeadline(t *testing.T) { + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: "weknoracloud", Model: "rerank", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + assert.Equal(t, 60, resolved.Rerank.RequestTimeout) + + // Everyone else has always run without one and relies on the caller. + for _, id := range []string{"zhipu", "jina", "siliconflow", "aliyun", "nvidia", "generic"} { + other, err := catalog.Resolve(catalog.Ref{ + Provider: id, Model: "m", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + assert.Zero(t, other.Rerank.RequestTimeout, "%s did not have a client deadline before", id) + } +} + +// TestTruncatePromptTokensIsReachableFromTheEditor closes the loop between the +// vendor accepting the extension and an operator being able to turn it on. +// Before the catalog it was readable from extra_config but had no input in the +// model editor, so the only way to set it was the API or the database. +// GET /models/providers renders extra fields dynamically, so declaring one is +// all it takes — and it must be declared exactly where the extension is +// accepted, or the form offers a switch the vendor will reject. +func TestTruncatePromptTokensIsReachableFromTheEditor(t *testing.T) { + for _, v := range catalog.ListByType(types.ModelTypeRerank) { + var field *catalog.ExtraField + for i := range v.ExtraFields { + if v.ExtraFields[i].Key == catalog.ExtraTruncatePromptTokens { + field = &v.ExtraFields[i] + } + } + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: v.ID, Model: "m", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + + if !resolved.Rerank.AcceptsTruncatePromptTokens { + assert.Nil(t, field, "%s does not accept the extension, so it must not offer the input", v.ID) + continue + } + require.NotNil(t, field, "%s accepts the extension but offers no way to set it", v.ID) + assert.Equal(t, "number", field.Type) + assert.False(t, field.Required, "the extension is opt-in") + assert.Equal(t, []types.ModelType{types.ModelTypeRerank}, field.ModelTypes, + "the input belongs to the rerank form only") + } +} + +// TestRerankScoreScalesMatchTheVendorDocs pins which vendors return something +// other than a 0..1 relevance score. Getting this wrong is silent: the number +// still looks like a score, and the retrieval threshold still compares it, so +// a logit-scaled vendor simply loses every negatively scored document. +func TestRerankScoreScalesMatchTheVendorDocs(t *testing.T) { + // Both spell the field like a probability and return neither. + // NIM: rankings[].logit, e.g. 0.226 / -1.171 / -6.31 + // GPUStack: relevance_score, e.g. 1.951 / -3.734 / -6.157 + logitScaled := map[string]bool{"nvidia": true, "gpustack": true} + + for _, v := range catalog.ListByType(types.ModelTypeRerank) { + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: v.ID, Model: "m", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + want := api.ScoreProbability + if logitScaled[v.ID] { + want = api.ScoreLogit + } + assert.Equal(t, want, resolved.Rerank.ScoreScale, "%s score scale", v.ID) + } +} + +// TestUnimplementedRerankDialectsAreRefused covers a model that names a +// protocol no package implements. Hiding it from the picker stops new rows; +// a row that already names it has to fail somewhere, and failing at +// construction with the reason beats sending a request shaped for the wrong +// protocol and reporting whatever the decoder makes of the reply. +func TestUnimplementedRerankDialectsAreRefused(t *testing.T) { + v, ok := catalog.Get("aliyun") + require.True(t, ok) + + offered := make([]string, 0) + for _, m := range v.ModelsByType(types.ModelTypeRerank) { + offered = append(offered, m.ID) + } + assert.NotContains(t, offered, "qwen3-rerank", "the picker must not offer an unimplemented dialect") + assert.Contains(t, offered, "gte-rerank-v2") + + _, err := catalog.Resolve(catalog.Ref{ + Provider: "aliyun", Model: "qwen3-rerank", ModelType: types.ModelTypeRerank, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "/compatible-api/v1/reranks", + "the error should name the protocol the model actually speaks") + + // The vendor's other rerank model is unaffected. + _, err = catalog.Resolve(catalog.Ref{ + Provider: "aliyun", Model: "gte-rerank-v2", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) +} + +// TestGatewayRerankEndpoints pins the URL each gateway's rerank rows reach. +// All three serve the Cohere dialect the protocol package already speaks, so +// opening the type was a declaration — but LiteLLM documents the route on the +// proxy root rather than under /v1, which is why it needs its own default. +func TestGatewayRerankEndpoints(t *testing.T) { + for _, tc := range []struct { + provider string + wantURL string + }{ + // https://docs.novita.ai/api-reference/model-apis-llm-create-rerank + {provider: "novita", wantURL: "https://api.novita.ai/openai/v1/rerank"}, + // https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request + {provider: "openrouter", wantURL: "https://openrouter.ai/api/v1/rerank"}, + // https://docs.litellm.ai/docs/rerank — the curl example posts to + // http://0.0.0.0:4000/rerank, not /v1/rerank. + {provider: "litellm", wantURL: "http://your_litellm_proxy/rerank"}, + } { + t.Run(tc.provider, func(t *testing.T) { + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: tc.provider, Model: "m", ModelType: types.ModelTypeRerank, + }) + require.NoError(t, err) + require.Equal(t, api.RerankCohere, resolved.RerankAPI) + + // Rebuild the URL the way cohererank does. + got := strings.TrimRight(resolved.BaseURL, "/") + if !strings.HasSuffix(got, "/rerank") { + got += "/rerank" + } + assert.Equal(t, tc.wantURL, got) + }) + } +} + +// TestScoreScaleIsOverridablePerRow covers the gateways, where the vendor +// cannot know the answer: GPUStack and a generic endpoint serve whatever +// reranker was deployed behind them, and the two families disagree — BGE +// answers a 0..1 probability, Qwen3-Reranker an unbounded score. Converting a +// probability as if it were a logit squeezes every score into [0.5, 0.73], +// which makes a relevance threshold meaningless, so the operator needs a way +// to say which one is actually there. +func TestScoreScaleIsOverridablePerRow(t *testing.T) { + for _, id := range []string{"gpustack", "generic"} { + t.Run(id, func(t *testing.T) { + base := "http://127.0.0.1:9/v1" + overridden, err := catalog.Resolve(catalog.Ref{ + Provider: id, Model: "bge-reranker-v2-m3", ModelType: types.ModelTypeRerank, + BaseURL: base, Extra: map[string]string{catalog.ExtraScoreScale: "probability"}, + }) + require.NoError(t, err) + assert.Equal(t, api.ScoreProbability, overridden.Rerank.ScoreScale) + + // And the input to set it is rendered by the editor. + v, ok := catalog.Get(id) + require.True(t, ok) + var field *catalog.ExtraField + for i := range v.ExtraFields { + if v.ExtraFields[i].Key == catalog.ExtraScoreScale { + field = &v.ExtraFields[i] + } + } + require.NotNil(t, field, "%s must offer the override it needs", id) + assert.Equal(t, "select", field.Type) + assert.Len(t, field.Options, 2) + }) + } + + _, err := catalog.Resolve(catalog.Ref{ + Provider: "gpustack", Model: "m", ModelType: types.ModelTypeRerank, + BaseURL: "http://127.0.0.1:9/v1", + Extra: map[string]string{catalog.ExtraScoreScale: "sigmoid"}, + }) + require.Error(t, err, "an unknown scale must not be treated as a probability") +} + +// TestGatewayVendorsOfferTheScoreScaleOverride keeps the override where the +// question is open. A gateway cannot know which reranker is deployed behind +// it, so all three offer the input; the other vendors' scales are settled by +// their own documentation and must not offer a switch that only lets an +// operator get it wrong. +func TestGatewayVendorsOfferTheScoreScaleOverride(t *testing.T) { + gateways := map[string]bool{"generic": true, "gpustack": true, "litellm": true} + + for _, v := range catalog.ListByType(types.ModelTypeRerank) { + var field *catalog.ExtraField + for i := range v.ExtraFields { + if v.ExtraFields[i].Key == catalog.ExtraScoreScale { + field = &v.ExtraFields[i] + } + } + if !gateways[v.ID] { + assert.Nil(t, field, "%s: its documentation settles the scale", v.ID) + continue + } + require.NotNil(t, field, "%s is a gateway and needs the override", v.ID) + require.Len(t, field.Options, 2) + for _, opt := range field.Options { + assert.NotEmpty(t, opt.LocalizedLabel("zh-CN"), "%s: option %q has no label", v.ID, opt.Value) + assert.NotEqual(t, opt.Label, opt.LocalizedLabel("zh-CN"), + "%s: option %q is prose and needs a zh-CN label", v.ID, opt.Value) + } + assert.NotEqual(t, field.Placeholder, field.LocalizedPlaceholder("zh-CN"), + "%s: the placeholder is prose and needs a zh-CN variant", v.ID) + } +} + +// Every operator-facing string a vendor declares has to exist in both +// languages, or half the product reads English prose in a Chinese form. +func TestEveryExtraFieldProseIsLocalized(t *testing.T) { + for _, v := range catalog.List() { + for _, field := range v.ExtraFields { + if field.Placeholder != "" && strings.Contains(field.Placeholder, " ") { + assert.NotEqual(t, field.Placeholder, field.LocalizedPlaceholder("zh-CN"), + "%s/%s: placeholder is prose without a zh-CN variant", v.ID, field.Key) + } + for _, opt := range field.Options { + // An identifier (a region code, a version) is the same in + // every language; prose is not. + if strings.Contains(opt.Label, " ") { + assert.NotEqual(t, opt.Label, opt.LocalizedLabel("zh-CN"), + "%s/%s: option %q is prose without a zh-CN label", v.ID, field.Key, opt.Value) + } + } + } + } +} diff --git a/internal/models/rerank/aliyun_reranker.go b/internal/models/rerank/aliyun_reranker.go deleted file mode 100644 index 4f891660..00000000 --- a/internal/models/rerank/aliyun_reranker.go +++ /dev/null @@ -1,172 +0,0 @@ -package rerank - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/Tencent/WeKnora/internal/logger" - secutils "github.com/Tencent/WeKnora/internal/utils" -) - -// AliyunReranker implements a reranking system based on Aliyun DashScope models -type AliyunReranker struct { - modelName string // Name of the model used for reranking - modelID string // Unique identifier of the model - apiKey string // API key for authentication - baseURL string // Base URL for API requests - client *http.Client // HTTP client for making API requests - customHeaders map[string]string -} - -// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 -func (r *AliyunReranker) SetCustomHeaders(headers map[string]string) { - r.customHeaders = headers -} - -// AliyunRerankRequest represents a request to rerank documents using Aliyun DashScope API -type AliyunRerankRequest struct { - Model string `json:"model"` // Model to use for reranking - Input AliyunRerankInput `json:"input"` // Input containing query and documents - Parameters AliyunRerankParameters `json:"parameters"` // Parameters for the reranking -} - -// AliyunRerankInput contains the query and documents for reranking -type AliyunRerankInput struct { - Query string `json:"query"` // Query text to compare documents against - Documents []string `json:"documents"` // List of document texts to rerank -} - -// AliyunRerankParameters contains parameters for the reranking request -type AliyunRerankParameters struct { - ReturnDocuments bool `json:"return_documents"` // Whether to return documents in response - TopN int `json:"top_n"` // Number of top results to return -} - -// AliyunRerankResponse represents the response from Aliyun DashScope reranking request -type AliyunRerankResponse struct { - Output AliyunOutput `json:"output"` // Output containing results - Usage AliyunUsage `json:"usage"` // Token usage information -} - -// AliyunOutput contains the reranking results -type AliyunOutput struct { - Results []AliyunRankResult `json:"results"` // Ranked results with relevance scores -} - -// AliyunRankResult represents a single reranking result from Aliyun -type AliyunRankResult struct { - Document AliyunDocument `json:"document"` // Document information - Index int `json:"index"` // Original index of the document - RelevanceScore float64 `json:"relevance_score"` // Relevance score -} - -// AliyunDocument represents document information in Aliyun response -type AliyunDocument struct { - Text string `json:"text"` // Document text -} - -// AliyunUsage contains information about token usage in the Aliyun API request -type AliyunUsage struct { - TotalTokens int `json:"total_tokens"` // Total tokens consumed -} - -// NewAliyunReranker creates a new instance of Aliyun reranker with the provided configuration -func NewAliyunReranker(config *RerankerConfig) (*AliyunReranker, error) { - apiKey := config.APIKey - baseURL := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank" - if url := config.BaseURL; url != "" { - baseURL = url - } - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - - return &AliyunReranker{ - modelName: config.ModelName, - modelID: config.ModelID, - apiKey: apiKey, - baseURL: baseURL, - client: newRerankHTTPClient(0), - }, nil -} - -// Rerank performs document reranking based on relevance to the query using Aliyun DashScope API -func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - // Build the request body - requestBody := &AliyunRerankRequest{ - Model: r.modelName, - Input: AliyunRerankInput{ - Query: query, - Documents: documents, - }, - Parameters: AliyunRerankParameters{ - ReturnDocuments: true, - TopN: len(documents), // Return all documents - }, - } - - jsonData, err := json.Marshal(requestBody) - if err != nil { - return nil, fmt.Errorf("marshal request body: %w", err) - } - - // Send the request - req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - secutils.ApplyCustomHeaders(req, r.customHeaders) - - logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents)) - - resp, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("do request: %w", err) - } - defer resp.Body.Close() - - // Read the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("aliyun rerank API error: Http Status: %s, Body: %s", resp.Status, string(body)) - } - - var response AliyunRerankResponse - if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("unmarshal response: %w", err) - } - - // Convert Aliyun results to standard RankResult format - results := make([]RankResult, len(response.Output.Results)) - for i, aliyunResult := range response.Output.Results { - results[i] = RankResult{ - Index: aliyunResult.Index, - Document: DocumentInfo{ - Text: aliyunResult.Document.Text, - }, - RelevanceScore: aliyunResult.RelevanceScore, - } - } - - return results, nil -} - -// GetModelName returns the name of the reranking model -func (r *AliyunReranker) GetModelName() string { - return r.modelName -} - -// GetModelID returns the unique identifier of the reranking model -func (r *AliyunReranker) GetModelID() string { - return r.modelID -} diff --git a/internal/models/rerank/jina_reranker.go b/internal/models/rerank/jina_reranker.go deleted file mode 100644 index c318765c..00000000 --- a/internal/models/rerank/jina_reranker.go +++ /dev/null @@ -1,128 +0,0 @@ -package rerank - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/Tencent/WeKnora/internal/logger" - secutils "github.com/Tencent/WeKnora/internal/utils" -) - -// JinaReranker implements a reranking system using Jina AI API -// Jina API uses different parameters than standard OpenAI-compatible APIs -type JinaReranker struct { - modelName string // Name of the model used for reranking - modelID string // Unique identifier of the model - apiKey string // API key for authentication - baseURL string // Base URL for API requests - client *http.Client // HTTP client for making API requests - customHeaders map[string]string -} - -// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 -func (r *JinaReranker) SetCustomHeaders(headers map[string]string) { - r.customHeaders = headers -} - -// JinaRerankRequest represents a Jina rerank request -// Note: Jina does NOT support truncate_prompt_tokens parameter -type JinaRerankRequest struct { - Model string `json:"model"` // Model to use for reranking - Query string `json:"query"` // Query text to compare documents against - Documents []string `json:"documents"` // List of document texts to rerank - TopN int `json:"top_n,omitempty"` // Number of top results to return - ReturnDocuments bool `json:"return_documents,omitempty"` // Whether to return document text in response -} - -// JinaRerankResponse represents the response from a Jina reranking request -type JinaRerankResponse struct { - Model string `json:"model"` // Model used for reranking - Results []RankResult `json:"results"` // Ranked results with relevance scores - Usage struct { - TotalTokens int `json:"total_tokens"` // Total tokens consumed - } `json:"usage"` -} - -// NewJinaReranker creates a new instance of Jina reranker with the provided configuration -func NewJinaReranker(config *RerankerConfig) (*JinaReranker, error) { - apiKey := config.APIKey - baseURL := "https://api.jina.ai/v1" - if url := config.BaseURL; url != "" { - baseURL = url - } - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - - return &JinaReranker{ - modelName: config.ModelName, - modelID: config.ModelID, - apiKey: apiKey, - baseURL: baseURL, - client: newRerankHTTPClient(0), - }, nil -} - -// Rerank performs document reranking based on relevance to the query -func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - // Build the request body - Jina does NOT use truncate_prompt_tokens - requestBody := &JinaRerankRequest{ - Model: r.modelName, - Query: query, - Documents: documents, - ReturnDocuments: true, // Return document text in response - } - - jsonData, err := json.Marshal(requestBody) - if err != nil { - return nil, fmt.Errorf("marshal request body: %w", err) - } - - // Send the request - req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - secutils.ApplyCustomHeaders(req, r.customHeaders) - - logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents)) - - resp, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("do request: %w", err) - } - defer resp.Body.Close() - - // Read the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body)) - return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status) - } - - var response JinaRerankResponse - if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("unmarshal response: %w", err) - } - return response.Results, nil -} - -// GetModelName returns the name of the reranking model -func (r *JinaReranker) GetModelName() string { - return r.modelName -} - -// GetModelID returns the unique identifier of the reranking model -func (r *JinaReranker) GetModelID() string { - return r.modelID -} diff --git a/internal/models/rerank/lkeap_reranker.go b/internal/models/rerank/lkeap_reranker.go index b9924bc4..f5c58cd3 100644 --- a/internal/models/rerank/lkeap_reranker.go +++ b/internal/models/rerank/lkeap_reranker.go @@ -4,19 +4,15 @@ import ( "context" "fmt" "strings" - "unicode/utf8" - "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" lkeap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lkeap/v20240522" ) const ( - // LKEAPMaxDocumentsPerRequest is the maximum number of documents accepted by RunRerank. - LKEAPMaxDocumentsPerRequest = 60 - // LKEAPMaxRequestCharacters is the maximum combined length of Query and Docs accepted by RunRerank. - LKEAPMaxRequestCharacters = 2000 // LKEAPRerankEndpoint 腾讯云知识引擎原子能力 Rerank API 域名 LKEAPRerankEndpoint = "lkeap.tencentcloudapi.com" // LKEAPDefaultRegion RunRerank 支持的地域,默认广州 @@ -25,16 +21,17 @@ const ( LKEAPDefaultRerankModel = "lke-reranker-base" ) -// LKEAPReranker 使用腾讯云知识引擎原子能力 RunRerank 接口进行重排序。 -// 鉴权使用腾讯云 API 密钥:APIKey 为 SecretId,AppSecret 为 SecretKey。 -type LKEAPReranker struct { +// lkeapClient calls Tencent Cloud's RunRerank action through the official +// SDK, which owns the TC3 signing this API requires. It implements +// api.Reranker and nothing else: the per-request ceilings (60 documents, +// 2000 characters for Query plus Docs together) are declared on the vendor +// and enforced by protocolReranker, like every other vendor's. +type lkeapClient struct { modelName string - modelID string client *lkeap.Client } -// NewLKEAPReranker 创建 LKEAP rerank 客户端。 -func NewLKEAPReranker(config *RerankerConfig) (*LKEAPReranker, error) { +func newLKEAPClient(config *RerankerConfig, resolved *catalog.Resolved) (api.Reranker, error) { secretID := strings.TrimSpace(config.APIKey) secretKey := strings.TrimSpace(config.AppSecret) if secretKey == "" && config.ExtraConfig != nil { @@ -60,86 +57,23 @@ func NewLKEAPReranker(config *RerankerConfig) (*LKEAPReranker, error) { return nil, fmt.Errorf("create LKEAP client: %w", err) } - modelName := strings.TrimSpace(config.ModelName) + modelName := strings.TrimSpace(resolved.RemoteModel) if modelName == "" { modelName = LKEAPDefaultRerankModel } - - return &LKEAPReranker{ - modelName: modelName, - modelID: config.ModelID, - client: client, - }, nil + return &lkeapClient{modelName: modelName, client: client}, nil } -// Rerank 调用 RunRerank 对文档按与 query 的相关性打分。 -func (r *LKEAPReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - if len(documents) == 0 { - return []RankResult{}, nil - } - - batches, err := lkeapRerankBatches(query, documents) - if err != nil { - return nil, err - } - - results := make([]RankResult, 0, len(documents)) - for _, batch := range batches { - batchResults, err := r.rerankBatch(ctx, query, batch.documents) - if err != nil { - return nil, err - } - for i := range batchResults { - batchResults[i].Index += batch.start - } - results = append(results, batchResults...) - } - return results, nil -} - -type lkeapRerankBatch struct { - start int - documents []string -} - -func lkeapRerankBatches(query string, documents []string) ([]lkeapRerankBatch, error) { - queryLength := utf8.RuneCountInString(query) - if queryLength >= LKEAPMaxRequestCharacters { - return nil, fmt.Errorf("LKEAP rerank query is %d characters; Query and Docs together support at most %d characters", queryLength, LKEAPMaxRequestCharacters) - } - - batches := make([]lkeapRerankBatch, 0, (len(documents)+LKEAPMaxDocumentsPerRequest-1)/LKEAPMaxDocumentsPerRequest) - batchStart := 0 - batchLength := queryLength - batchDocuments := make([]string, 0, LKEAPMaxDocumentsPerRequest) - for index, document := range documents { - documentLength := utf8.RuneCountInString(document) - if queryLength+documentLength > LKEAPMaxRequestCharacters { - return nil, fmt.Errorf("LKEAP rerank document at index %d is %d characters; Query and each document together support at most %d characters", index, documentLength, LKEAPMaxRequestCharacters) - } - if len(batchDocuments) == LKEAPMaxDocumentsPerRequest || batchLength+documentLength > LKEAPMaxRequestCharacters { - batches = append(batches, lkeapRerankBatch{start: batchStart, documents: batchDocuments}) - batchStart = index - batchLength = queryLength - batchDocuments = make([]string, 0, LKEAPMaxDocumentsPerRequest) - } - batchDocuments = append(batchDocuments, document) - batchLength += documentLength - } - if len(batchDocuments) > 0 { - batches = append(batches, lkeapRerankBatch{start: batchStart, documents: batchDocuments}) - } - return batches, nil -} - -func (r *LKEAPReranker) rerankBatch(ctx context.Context, query string, documents []string) ([]RankResult, error) { +// Rerank scores one batch. RunRerank answers a bare score list in input +// order rather than an indexed, sorted result set. +func (r *lkeapClient) Rerank( + ctx context.Context, query string, documents []string, +) ([]api.RerankResult, error) { req := lkeap.NewRunRerankRequest() req.Query = common.StringPtr(query) req.Docs = common.StringPtrs(documents) req.Model = common.StringPtr(r.modelName) - logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, LKEAPRerankEndpoint, query, documents)) - resp, err := r.client.RunRerankWithContext(ctx, req) if err != nil { return nil, fmt.Errorf("LKEAP RunRerank: %w", err) @@ -147,35 +81,18 @@ func (r *LKEAPReranker) rerankBatch(ctx context.Context, query string, documents if resp == nil || resp.Response == nil || len(resp.Response.ScoreList) == 0 { return nil, fmt.Errorf("LKEAP rerank API returned empty score list") } - scores := resp.Response.ScoreList if len(scores) != len(documents) { return nil, fmt.Errorf("LKEAP rerank score count mismatch: got %d scores for %d documents", len(scores), len(documents)) } - results := make([]RankResult, len(documents)) + results := make([]api.RerankResult, len(documents)) for i, score := range scores { - if score == nil { - continue - } - results[i] = RankResult{ - Index: i, - Document: DocumentInfo{ - Text: documents[i], - }, - RelevanceScore: *score, + results[i] = api.RerankResult{Index: i, Text: documents[i]} + if score != nil { + results[i].Score = *score } } return results, nil } - -// GetModelName returns the rerank model name. -func (r *LKEAPReranker) GetModelName() string { - return r.modelName -} - -// GetModelID returns the model ID. -func (r *LKEAPReranker) GetModelID() string { - return r.modelID -} diff --git a/internal/models/rerank/lkeap_reranker_test.go b/internal/models/rerank/lkeap_reranker_test.go deleted file mode 100644 index 467b0589..00000000 --- a/internal/models/rerank/lkeap_reranker_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package rerank - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "net/url" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" - "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" - lkeap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lkeap/v20240522" -) - -func TestNewLKEAPReranker_requiresCredentials(t *testing.T) { - _, err := NewLKEAPReranker(&RerankerConfig{ - ModelName: "lke-reranker-base", - }) - require.Error(t, err) - assert.Contains(t, err.Error(), "secret_id") -} - -func TestNewLKEAPReranker_secretKeyFromExtraConfig(t *testing.T) { - r, err := NewLKEAPReranker(&RerankerConfig{ - APIKey: "AKIDtest", - ModelName: "lke-reranker-base", - ExtraConfig: map[string]string{ - "secret_key": "sk-test", - "region": "ap-beijing", - }, - }) - require.NoError(t, err) - require.NotNil(t, r) - assert.Equal(t, "lke-reranker-base", r.GetModelName()) -} - -func TestNewLKEAPReranker_defaultModelName(t *testing.T) { - r, err := NewLKEAPReranker(&RerankerConfig{ - APIKey: "AKIDtest", - AppSecret: "sk-test", - }) - require.NoError(t, err) - assert.Equal(t, LKEAPDefaultRerankModel, r.GetModelName()) -} - -func TestLKEAPReranker_Rerank_emptyDocuments(t *testing.T) { - r, err := NewLKEAPReranker(&RerankerConfig{ - APIKey: "AKIDtest", - AppSecret: "sk-test", - }) - require.NoError(t, err) - - results, err := r.Rerank(t.Context(), "query", nil) - require.NoError(t, err) - assert.Empty(t, results) -} - -func TestLKEAPReranker_Rerank_batchesMoreThan60Documents(t *testing.T) { - var batches [][]string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - var request struct { - Docs []string `json:"Docs"` - } - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - batches = append(batches, request.Docs) - - scores := make([]float64, len(request.Docs)) - for i, doc := range request.Docs { - index, err := lkeapTestDocumentIndex(doc) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - scores[i] = float64(index) - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "Response": map[string]any{ - "ScoreList": scores, - "RequestId": "test-request", - }, - }) - })) - defer server.Close() - - r := newTestLKEAPReranker(t, server.URL) - documents := make([]string, 61) - for i := range documents { - documents[i] = strconv.Itoa(i) + ":document" - } - - results, err := r.Rerank(t.Context(), "query", documents) - require.NoError(t, err) - require.Len(t, results, len(documents)) - require.Len(t, batches, 2) - assert.Len(t, batches[0], 60) - assert.Len(t, batches[1], 1) - for i, result := range results { - assert.Equal(t, i, result.Index) - assert.Equal(t, documents[i], result.Document.Text) - assert.Equal(t, float64(i), result.RelevanceScore) - } -} - -func TestLKEAPReranker_Rerank_batchesWithinCharacterLimit(t *testing.T) { - var batches [][]string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - var request struct { - Docs []string `json:"Docs"` - } - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - batches = append(batches, request.Docs) - - scores := make([]float64, len(request.Docs)) - for i, doc := range request.Docs { - index, err := lkeapTestDocumentIndex(doc) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - scores[i] = float64(index) - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "Response": map[string]any{ - "ScoreList": scores, - "RequestId": "test-request", - }, - }) - })) - defer server.Close() - - r := newTestLKEAPReranker(t, server.URL) - documents := []string{ - "0:" + strings.Repeat("a", 1000), - "1:" + strings.Repeat("b", 1000), - "2:" + strings.Repeat("c", 1000), - } - - results, err := r.Rerank(t.Context(), "query", documents) - require.NoError(t, err) - require.Len(t, results, len(documents)) - require.Len(t, batches, 3) - for i, batch := range batches { - assert.Len(t, batch, 1) - assert.Equal(t, documents[i], batch[0]) - } -} - -func newTestLKEAPReranker(t *testing.T, serverURL string) *LKEAPReranker { - t.Helper() - endpoint, err := url.Parse(serverURL) - require.NoError(t, err) - - clientProfile := profile.NewClientProfile() - clientProfile.HttpProfile.Endpoint = endpoint.Host - clientProfile.HttpProfile.Scheme = "HTTP" - client, err := lkeap.NewClient(common.NewCredential("AKIDtest", "sk-test"), LKEAPDefaultRegion, clientProfile) - require.NoError(t, err) - - return &LKEAPReranker{ - modelName: LKEAPDefaultRerankModel, - client: client, - } -} - -func lkeapTestDocumentIndex(document string) (int, error) { - value, _, _ := strings.Cut(document, ":") - return strconv.Atoi(value) -} diff --git a/internal/models/rerank/nvidia_reranker.go b/internal/models/rerank/nvidia_reranker.go deleted file mode 100644 index cfc7360b..00000000 --- a/internal/models/rerank/nvidia_reranker.go +++ /dev/null @@ -1,156 +0,0 @@ -package rerank - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "math" - "net/http" - - "github.com/Tencent/WeKnora/internal/logger" - secutils "github.com/Tencent/WeKnora/internal/utils" -) - -// NvidiaReranker implements a reranking system using Jina AI API -// Jina API uses different parameters than standard OpenAI-compatible APIs -type NvidiaReranker struct { - modelName string // Name of the model used for reranking - modelID string // Unique identifier of the model - apiKey string // API key for authentication - baseURL string // Base URL for API requests - client *http.Client // HTTP client for making API requests - customHeaders map[string]string -} - -// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 -func (r *NvidiaReranker) SetCustomHeaders(headers map[string]string) { - r.customHeaders = headers -} - -type NvidiaRerankDocument struct { - Text string `json:"text"` -} - -// NvidiaRerankRequest represents a Jina rerank request -// Note: Jina does NOT support truncate_prompt_tokens parameter -type NvidiaRerankRequest struct { - Model string `json:"model"` // Model to use for reranking - Query NvidiaRerankDocument `json:"query"` // Query text to compare documents against - Documents []NvidiaRerankDocument `json:"passages"` // List of document texts to rerank -} - -type NvidiaRankResult struct { - Index int `json:"index"` - Logit float64 `json:"logit"` -} - -// NvidiaRerankResponse represents the response from a Jina reranking request -type NvidiaRerankResponse struct { - Model string `json:"model"` // Model used for reranking - Results []NvidiaRankResult `json:"rankings"` // Ranked results with relevance scores -} - -// NewNvidiaReranker creates a new instance of Jina reranker with the provided configuration -func NewNvidiaReranker(config *RerankerConfig) (*NvidiaReranker, error) { - apiKey := config.APIKey - baseURL := "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking" - if url := config.BaseURL; url != "" { - baseURL = url - } - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - - return &NvidiaReranker{ - modelName: config.ModelName, - modelID: config.ModelID, - apiKey: apiKey, - baseURL: baseURL, - client: newRerankHTTPClient(0), - }, nil -} - -// Rerank performs document reranking based on relevance to the query -func (r *NvidiaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - // Build the request body - Jina does NOT use truncate_prompt_tokens - requestBody := &NvidiaRerankRequest{ - Model: r.modelName, - Query: NvidiaRerankDocument{Text: query}, - Documents: make([]NvidiaRerankDocument, len(documents)), - } - for i := range requestBody.Documents { - requestBody.Documents[i].Text = documents[i] - } - - jsonData, err := json.Marshal(requestBody) - if err != nil { - return nil, fmt.Errorf("marshal request body: %w", err) - } - - // Send the request - req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - secutils.ApplyCustomHeaders(req, r.customHeaders) - - // Log the curl equivalent for debugging (API key masked for security) - logger.GetLogger(ctx).Infof( - "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'", - r.baseURL, string(jsonData), - ) - - resp, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("do request: %w", err) - } - defer resp.Body.Close() - - // Read the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body)) - return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status) - } - - var response NvidiaRerankResponse - if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("unmarshal response: %w", err) - } - ret := make([]RankResult, len(response.Results)) - for i, result := range response.Results { - ret[i] = RankResult{ - Index: result.Index, - Document: DocumentInfo{Text: documents[result.Index]}, - RelevanceScore: normalizeNvidiaLogit(result.Logit), - } - } - return ret, nil -} - -// normalizeNvidiaLogit converts NVIDIA's raw reranker logit into a probability. -func normalizeNvidiaLogit(logit float64) float64 { - if logit >= 0 { - return 1 / (1 + math.Exp(-logit)) - } - expLogit := math.Exp(logit) - return expLogit / (1 + expLogit) -} - -// GetModelName returns the name of the reranking model -func (r *NvidiaReranker) GetModelName() string { - return r.modelName -} - -// GetModelID returns the unique identifier of the reranking model -func (r *NvidiaReranker) GetModelID() string { - return r.modelID -} diff --git a/internal/models/rerank/nvidia_reranker_test.go b/internal/models/rerank/nvidia_reranker_test.go deleted file mode 100644 index e93e4b6e..00000000 --- a/internal/models/rerank/nvidia_reranker_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package rerank - -import ( - "math" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNvidiaRerankerNormalizesLogitsToProbabilities(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - require.Equal(t, http.MethodPost, r.Method) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "model": "nvidia-rerank-model", - "rankings": [ - {"index": 0, "logit": 23.0}, - {"index": 1, "logit": 0.0}, - {"index": 2, "logit": -23.0} - ] - }`)) - })) - defer server.Close() - - reranker := &NvidiaReranker{ - modelName: "nvidia-rerank-model", - apiKey: "nvapi-test", - baseURL: server.URL, - client: server.Client(), - } - - results, err := reranker.Rerank(t.Context(), "query", []string{"high", "neutral", "low"}) - require.NoError(t, err) - require.Len(t, results, 3) - - logits := []float64{23, 0, -23} - for i, logit := range logits { - want := 1 / (1 + math.Exp(-logit)) - assert.InDelta(t, want, results[i].RelevanceScore, 1e-12) - assert.Greater(t, results[i].RelevanceScore, 0.0) - assert.Less(t, results[i].RelevanceScore, 1.0) - } - - assert.Greater(t, results[0].RelevanceScore, results[1].RelevanceScore) - assert.Greater(t, results[1].RelevanceScore, results[2].RelevanceScore) -} diff --git a/internal/models/rerank/protocol.go b/internal/models/rerank/protocol.go new file mode 100644 index 00000000..63d580b5 --- /dev/null +++ b/internal/models/rerank/protocol.go @@ -0,0 +1,142 @@ +package rerank + +import ( + "context" + "fmt" + "math" + "sort" + "unicode/utf8" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "golang.org/x/sync/errgroup" +) + +// protocolReranker adapts a protocol client to the Reranker interface and +// owns the two things every vendor needs and none of them should implement +// itself: splitting a candidate set that exceeds the documented per-request +// ceilings, and putting the returned scores on one scale. +type protocolReranker struct { + inner api.Reranker + settings catalog.RerankSettings + endpoint string + modelName string + modelID string +} + +// defaultBatchConcurrency bounds in-flight batches when a vendor declares no +// preference, so a large embedding_top_k cannot fan out into an unbounded +// burst of requests. +const defaultBatchConcurrency = 4 + +func (r *protocolReranker) GetModelName() string { return r.modelName } +func (r *protocolReranker) GetModelID() string { return r.modelID } + +func (r *protocolReranker) Rerank( + ctx context.Context, query string, documents []string, +) ([]RankResult, error) { + if len(documents) == 0 { + return nil, nil + } + logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.endpoint, query, documents)) + + // The query travels in every request, so an over-long one cannot be made + // to fit by splitting the documents. SplitBatches charges it to each + // batch but only against a whole-request budget; a vendor that caps the + // query on its own is checked here. + if limit := r.settings.MaxQueryChars; limit > 0 { + if length := utf8.RuneCountInString(query); length > limit { + return nil, fmt.Errorf( + "%s rerank: query is %d characters; the limit is %d", + r.modelName, length, limit, + ) + } + } + + batches, err := api.SplitBatches(documents, utf8.RuneCountInString(query), r.settings.BatchLimits()) + if err != nil { + return nil, fmt.Errorf("%s rerank: %w", r.modelName, err) + } + + // Each batch is scored against the same query independently, so the + // per-batch results are comparable and are merged and re-ranked below. + scored := make([][]api.RerankResult, len(batches)) + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(r.concurrency()) + for i, batch := range batches { + group.Go(func() error { + out, err := r.inner.Rerank(groupCtx, query, batch.Items) + if err != nil { + return err + } + for j := range out { + out[j].Index += batch.Start + } + scored[i] = out + return nil + }) + } + if err := group.Wait(); err != nil { + return nil, err + } + + results := make([]RankResult, 0, len(documents)) + for _, batch := range scored { + for _, item := range batch { + if item.Index < 0 || item.Index >= len(documents) { + return nil, fmt.Errorf( + "%s rerank: index %d out of range for %d documents", + r.modelName, item.Index, len(documents), + ) + } + text := item.Text + if text == "" { + text = documents[item.Index] + } + results = append(results, RankResult{ + Index: item.Index, + Document: DocumentInfo{Text: text}, + RelevanceScore: normalizeScore(item.Score, r.settings.ScoreScale), + }) + } + } + // A single request comes back ranked, and the retrieval pipeline reads + // the slice as ranked: it takes results[0] as the best candidate for its + // threshold fallback. Concatenating per-batch results would make that the + // best of the first batch only, so the merged set is ordered once here. + // Ties keep the lower index, which keeps the order stable. + sort.SliceStable(results, func(i, j int) bool { + return results[i].RelevanceScore > results[j].RelevanceScore + }) + return results, nil +} + +func (r *protocolReranker) concurrency() int { + if r.settings.MaxConcurrency > 0 { + return r.settings.MaxConcurrency + } + return defaultBatchConcurrency +} + +// normalizeScore puts every vendor's score on the 0..1 scale the retrieval +// pipeline compares against RerankThreshold. +// +// Most protocols already return a relevance probability. NIM returns the raw +// logit of its relevance head instead — unbounded and routinely negative — +// so a threshold tuned for probabilities would reject almost everything on +// that vendor. The logistic function is the inverse of a log-odds, so this is +// a unit conversion rather than a heuristic: it maps the vendor's own ranking +// onto the scale the rest of the system already speaks, order preserved. +func normalizeScore(score float64, scale api.ScoreScale) float64 { + if scale != api.ScoreLogit { + return score + } + // Two branches so neither exponential overflows: this is the form the + // pre-catalog NVIDIA client used, kept verbatim. + if score >= 0 { + return 1 / (1 + math.Exp(-score)) + } + exp := math.Exp(score) + return exp / (1 + exp) +} diff --git a/internal/models/rerank/protocol_test.go b/internal/models/rerank/protocol_test.go new file mode 100644 index 00000000..621256a9 --- /dev/null +++ b/internal/models/rerank/protocol_test.go @@ -0,0 +1,187 @@ +package rerank + +import ( + "context" + "fmt" + "math" + "strings" + "sync" + "testing" + + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeProtocol records every batch it was handed and scores documents by +// their position within the batch, so a wrong offset shows up immediately. +type fakeProtocol struct { + mu sync.Mutex + batches [][]string + err error +} + +func (f *fakeProtocol) Rerank( + _ context.Context, _ string, documents []string, +) ([]api.RerankResult, error) { + f.mu.Lock() + f.batches = append(f.batches, append([]string(nil), documents...)) + f.mu.Unlock() + if f.err != nil { + return nil, f.err + } + out := make([]api.RerankResult, len(documents)) + for i, doc := range documents { + out[i] = api.RerankResult{Index: i, Score: float64(i), Text: doc} + } + return out, nil +} + +func newWrapped(inner api.Reranker, settings catalog.RerankSettings) *protocolReranker { + return &protocolReranker{ + inner: inner, settings: settings, + endpoint: "https://example.invalid", modelName: "m", modelID: "id", + } +} + +// The documented per-request ceilings live on the vendor, and one shared +// layer enforces them. Indices must come back pointing into the caller's +// original slice, not into the batch. +func TestBatchingSplitsAndRestoresGlobalIndices(t *testing.T) { + fake := &fakeProtocol{} + r := newWrapped(fake, catalog.RerankSettings{MaxDocuments: 2, MaxConcurrency: 1}) + + docs := []string{"d0", "d1", "d2", "d3", "d4"} + got, err := r.Rerank(context.Background(), "q", docs) + require.NoError(t, err) + + require.Len(t, fake.batches, 3, "5 documents at 2 per request is 3 requests") + indices := make([]int, 0, len(got)) + for _, item := range got { + indices = append(indices, item.Index) + assert.Equal(t, docs[item.Index], item.Document.Text, "index must address the caller's slice") + } + assert.ElementsMatch(t, []int{0, 1, 2, 3, 4}, indices) +} + +func TestBatchingIsSkippedWhenTheVendorDeclaresNoLimit(t *testing.T) { + fake := &fakeProtocol{} + r := newWrapped(fake, catalog.RerankSettings{}) + + _, err := r.Rerank(context.Background(), "q", []string{"a", "b", "c", "d"}) + require.NoError(t, err) + assert.Len(t, fake.batches, 1) +} + +// A document over the vendor's per-document ceiling cannot be scored. Saying +// so beats sending a prefix and returning a number for text nobody asked +// about. +func TestAnOversizedDocumentIsAnError(t *testing.T) { + r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxDocumentChars: 10}) + + _, err := r.Rerank(context.Background(), "q", []string{"ok", strings.Repeat("x", 99)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "index 1") +} + +func TestBatchErrorsPropagate(t *testing.T) { + r := newWrapped(&fakeProtocol{err: fmt.Errorf("upstream exploded")}, catalog.RerankSettings{}) + + _, err := r.Rerank(context.Background(), "q", []string{"a"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "upstream exploded") +} + +func TestEmptyInputMakesNoRequest(t *testing.T) { + fake := &fakeProtocol{} + r := newWrapped(fake, catalog.RerankSettings{}) + + got, err := r.Rerank(context.Background(), "q", nil) + require.NoError(t, err) + assert.Empty(t, got) + assert.Empty(t, fake.batches) +} + +// NIM returns the raw logit of its relevance head. The retrieval pipeline +// compares scores against RerankThreshold, which is tuned for probabilities, +// so the logit is converted rather than passed through — otherwise the +// default threshold of 0.2 keeps only documents whose logit is positive. +func TestLogitScoresBecomeProbabilities(t *testing.T) { + for _, tc := range []struct { + logit float64 + want float64 + }{ + {logit: 0.226318359375, want: 0.5563}, + {logit: -1.171875, want: 0.2366}, + {logit: -6.3125, want: 0.0018}, + {logit: 0, want: 0.5}, + } { + assert.InDelta(t, tc.want, normalizeScore(tc.logit, api.ScoreLogit), 1e-4, + "logit %v", tc.logit) + } +} + +// The conversion is monotonic, so it re-scales without reordering. +func TestLogitConversionPreservesOrder(t *testing.T) { + logits := []float64{-6.3125, -1.171875, 0.226318359375, 4} + previous := -1.0 + for _, logit := range logits { + got := normalizeScore(logit, api.ScoreLogit) + assert.Greater(t, got, previous) + previous = got + } +} + +func TestProbabilityScoresPassThroughUntouched(t *testing.T) { + assert.Equal(t, 0.9819, normalizeScore(0.9819, api.ScoreProbability)) + assert.Equal(t, 0.9819, normalizeScore(0.9819, "")) +} + +// A vendor that caps the query cannot be satisfied by splitting documents, +// so the query is checked on its own before any request goes out. +func TestAnOversizedQueryIsAnError(t *testing.T) { + fake := &fakeProtocol{} + r := newWrapped(fake, catalog.RerankSettings{MaxQueryChars: 10}) + + _, err := r.Rerank(context.Background(), strings.Repeat("q", 11), []string{"d"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "query is 11 characters") + assert.Empty(t, fake.batches, "nothing should reach the vendor") +} + +func TestAQueryWithinTheLimitPasses(t *testing.T) { + r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxQueryChars: 10}) + _, err := r.Rerank(context.Background(), strings.Repeat("中", 10), []string{"d"}) + require.NoError(t, err, "runes, not bytes") +} + +// The retrieval pipeline reads results[0] as the best candidate. Batches come +// back each individually ranked, so the merged set has to be re-ranked or the +// caller would see the best of the first batch only. +func TestMergedBatchesAreRankedGlobally(t *testing.T) { + // Scores are the position within the batch, so batch 2 holds the highest. + r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxDocuments: 2, MaxConcurrency: 1}) + + got, err := r.Rerank(context.Background(), "q", []string{"d0", "d1", "d2", "d3", "d4"}) + require.NoError(t, err) + require.Len(t, got, 5) + + for i := 1; i < len(got); i++ { + assert.GreaterOrEqual(t, got[i-1].RelevanceScore, got[i].RelevanceScore, + "results must be ranked across batches, not concatenated") + } +} + +// The pre-catalog NVIDIA client used a two-branch logistic so neither +// exponential overflows. These are the values its own test pinned. +func TestLogitConversionHandlesExtremes(t *testing.T) { + for _, logit := range []float64{23, 0, -23} { + got := normalizeScore(logit, api.ScoreLogit) + assert.InDelta(t, 1/(1+math.Exp(-logit)), got, 1e-12, "logit %v", logit) + assert.Greater(t, got, 0.0) + assert.Less(t, got, 1.0) + } + assert.False(t, math.IsNaN(normalizeScore(-1000, api.ScoreLogit))) + assert.False(t, math.IsNaN(normalizeScore(1000, api.ScoreLogit))) +} diff --git a/internal/models/rerank/remote_api.go b/internal/models/rerank/remote_api.go deleted file mode 100644 index fc0feb14..00000000 --- a/internal/models/rerank/remote_api.go +++ /dev/null @@ -1,153 +0,0 @@ -package rerank - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "strings" - - "github.com/Tencent/WeKnora/internal/logger" - secutils "github.com/Tencent/WeKnora/internal/utils" -) - -// OpenAIReranker implements a reranking system based on OpenAI models -type OpenAIReranker struct { - modelName string // Name of the model used for reranking - modelID string // Unique identifier of the model - apiKey string // API key for authentication - baseURL string // Base URL for API requests - client *http.Client // HTTP client for making API requests - customHeaders map[string]string - // truncatePromptTokens, when > 0, is sent as the vLLM-specific - // truncate_prompt_tokens request field. It must never be sent by default: - // providers that honor it (e.g. SiliconFlow) keep only the LAST N tokens of - // the templated rerank prompt, which cuts the query off long documents and - // collapses every relevance score to near zero (issue #2143). - truncatePromptTokens int -} - -// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 -func (r *OpenAIReranker) SetCustomHeaders(headers map[string]string) { - r.customHeaders = headers -} - -// RerankRequest represents a request to rerank documents based on relevance to a query -type RerankRequest struct { - Model string `json:"model"` // Model to use for reranking - Query string `json:"query"` // Query text to compare documents against - Documents []string `json:"documents"` // List of document texts to rerank - AdditionalData map[string]interface{} `json:"additional_data,omitempty"` // Optional additional data for the model - TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` // Maximum prompt tokens to use (vLLM-specific, opt-in) -} - -// RerankResponse represents the response from a reranking request -type RerankResponse struct { - ID string `json:"id"` // Request ID - Model string `json:"model"` // Model used for reranking - Usage UsageInfo `json:"usage"` // Token usage information - Results []RankResult `json:"results"` // Ranked results with relevance scores -} - -// UsageInfo contains information about token usage in the API request -type UsageInfo struct { - TotalTokens int `json:"total_tokens"` // Total tokens consumed -} - -// NewOpenAIReranker creates a new instance of OpenAI reranker with the provided configuration -func NewOpenAIReranker(config *RerankerConfig) (*OpenAIReranker, error) { - apiKey := config.APIKey - baseURL := "https://api.openai.com/v1" - if url := config.BaseURL; url != "" { - baseURL = url - } - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - - // Optional opt-in for vLLM-style deployments that need server-side prompt - // truncation. Configured via extra_config; never enabled by default. - truncatePromptTokens := 0 - if config.ExtraConfig != nil { - if raw := strings.TrimSpace(config.ExtraConfig["truncate_prompt_tokens"]); raw != "" { - n, err := strconv.Atoi(raw) - if err != nil || n <= 0 { - return nil, fmt.Errorf("invalid truncate_prompt_tokens in extra_config: %q", raw) - } - truncatePromptTokens = n - } - } - - return &OpenAIReranker{ - modelName: config.ModelName, - modelID: config.ModelID, - apiKey: apiKey, - baseURL: baseURL, - client: newRerankHTTPClient(0), - truncatePromptTokens: truncatePromptTokens, - }, nil -} - -// Rerank performs document reranking based on relevance to the query -func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - // Build the request body. truncate_prompt_tokens is only included when - // explicitly configured: sending it unconditionally corrupts scores on - // providers that honor it (see OpenAIReranker.truncatePromptTokens). - requestBody := &RerankRequest{ - Model: r.modelName, - Query: query, - Documents: documents, - TruncatePromptTokens: r.truncatePromptTokens, - } - - jsonData, err := json.Marshal(requestBody) - if err != nil { - return nil, fmt.Errorf("marshal request body: %w", err) - } - - // Send the request - req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - secutils.ApplyCustomHeaders(req, r.customHeaders) - - logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents)) - - resp, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("do request: %w", err) - } - defer resp.Body.Close() - - // Read the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status) - } - - var response RerankResponse - if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("unmarshal response: %w", err) - } - return response.Results, nil -} - -// GetModelName returns the name of the reranking model -func (r *OpenAIReranker) GetModelName() string { - return r.modelName -} - -// GetModelID returns the unique identifier of the reranking model -func (r *OpenAIReranker) GetModelID() string { - return r.modelID -} diff --git a/internal/models/rerank/remote_api_test.go b/internal/models/rerank/remote_api_test.go deleted file mode 100644 index ef4fa548..00000000 --- a/internal/models/rerank/remote_api_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package rerank - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" -) - -// newRerankScoreTestServer emulates an OpenAI-compatible /rerank endpoint the -// way a vLLM-backed provider (e.g. SiliconFlow) behaves, and records the last -// decoded request body. -// -// If the request carries truncate_prompt_tokens, the backend keeps only the -// last N tokens of the templated rerank prompt — the query gets cut off long -// documents and every relevance score collapses to near zero (issue #2143). -// Otherwise it returns the real scores, sorted by relevance_score descending, -// with index pointing back at the original input-document position. -func newRerankScoreTestServer(t *testing.T, lastRequest *map[string]interface{}) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req map[string]interface{} - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("decode rerank request: %v", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - *lastRequest = req - - w.Header().Set("Content-Type", "application/json") - if _, truncated := req["truncate_prompt_tokens"]; truncated { - // Query truncated away from the prompt: scores collapse. - _, _ = w.Write([]byte(`{ - "id": "rerank-collapsed", - "results": [ - {"index": 0, "relevance_score": 0.00670681, "document": {"text": "A"}}, - {"index": 1, "relevance_score": 0.00587342, "document": {"text": "B"}}, - {"index": 2, "relevance_score": 0.00412907, "document": {"text": "C"}} - ], - "usage": {"total_tokens": 42} - }`)) - return - } - // Healthy response: sorted by relevance_score descending, index values - // out of order relative to the input documents [A, B, C]. - _, _ = w.Write([]byte(`{ - "id": "rerank-ok", - "results": [ - {"index": 2, "relevance_score": 0.998, "document": {"text": "C"}}, - {"index": 0, "relevance_score": 0.51, "document": {"text": "A"}}, - {"index": 1, "relevance_score": 0.006, "document": {"text": "B"}} - ], - "usage": {"total_tokens": 42} - }`)) - })) -} - -// TestOpenAIRerankerDoesNotTruncatePromptByDefault is the regression test for -// issue #2143: the generic OpenAI-compatible reranker must not send the -// vLLM-specific truncate_prompt_tokens field unless explicitly configured, and -// each input document must keep its own relevance score via the index field. -func TestOpenAIRerankerDoesNotTruncatePromptByDefault(t *testing.T) { - withRerankSSRFWhitelist(t, "127.0.0.1") - - var lastRequest map[string]interface{} - server := newRerankScoreTestServer(t, &lastRequest) - defer server.Close() - - reranker, err := NewOpenAIReranker(&RerankerConfig{ - BaseURL: server.URL, - ModelName: "Qwen/Qwen3-VL-Reranker-8B", - APIKey: "sk-test", - }) - if err != nil { - t.Fatalf("NewOpenAIReranker: %v", err) - } - - documents := []string{"A", "B", "C"} - results, err := reranker.Rerank(t.Context(), "query", documents) - if err != nil { - t.Fatalf("Rerank: %v", err) - } - - if v, ok := lastRequest["truncate_prompt_tokens"]; ok { - t.Errorf("request contains truncate_prompt_tokens=%v; it must not be sent unless configured", v) - } - if v, ok := lastRequest["additional_data"]; ok { - t.Errorf("request contains additional_data=%v; empty optional fields must be omitted", v) - } - - if len(results) != len(documents) { - t.Fatalf("got %d results, want %d", len(results), len(documents)) - } - - // Results stay in the order returned by the API (sorted by score desc). - if results[0].Index != 2 || results[0].RelevanceScore != 0.998 { - t.Errorf("top result = {index: %d, score: %v}, want {index: 2, score: 0.998}", - results[0].Index, results[0].RelevanceScore) - } - - // Each input document keeps its own score, resolved through the index field. - wantScoreByDoc := map[string]float64{"C": 0.998, "A": 0.51, "B": 0.006} - for _, rr := range results { - if rr.Index < 0 || rr.Index >= len(documents) { - t.Fatalf("result index %d out of range for %d documents", rr.Index, len(documents)) - } - doc := documents[rr.Index] - if want := wantScoreByDoc[doc]; rr.RelevanceScore != want { - t.Errorf("document %q got score %v, want %v", doc, rr.RelevanceScore, want) - } - } -} - -// TestOpenAIRerankerTruncatePromptTokensOptIn verifies that deployments which -// really need server-side prompt truncation (self-hosted vLLM with small- -// context rerankers) can still opt in via extra_config. -func TestOpenAIRerankerTruncatePromptTokensOptIn(t *testing.T) { - withRerankSSRFWhitelist(t, "127.0.0.1") - - var lastRequest map[string]interface{} - server := newRerankScoreTestServer(t, &lastRequest) - defer server.Close() - - reranker, err := NewOpenAIReranker(&RerankerConfig{ - BaseURL: server.URL, - ModelName: "bge-reranker-base", - APIKey: "sk-test", - ExtraConfig: map[string]string{"truncate_prompt_tokens": "511"}, - }) - if err != nil { - t.Fatalf("NewOpenAIReranker: %v", err) - } - - if _, err := reranker.Rerank(t.Context(), "query", []string{"A", "B", "C"}); err != nil { - t.Fatalf("Rerank: %v", err) - } - - v, ok := lastRequest["truncate_prompt_tokens"] - if !ok { - t.Fatal("request is missing truncate_prompt_tokens despite extra_config opt-in") - } - if n, _ := v.(float64); n != 511 { - t.Errorf("truncate_prompt_tokens = %v, want 511", v) - } -} - -func TestNewOpenAIRerankerRejectsInvalidTruncatePromptTokens(t *testing.T) { - for _, raw := range []string{"abc", "-1", "0"} { - _, err := NewOpenAIReranker(&RerankerConfig{ - ModelName: "rerank-test", - ExtraConfig: map[string]string{"truncate_prompt_tokens": raw}, - }) - if err == nil { - t.Errorf("NewOpenAIReranker accepted invalid truncate_prompt_tokens %q", raw) - } - } -} diff --git a/internal/models/rerank/reranker.go b/internal/models/rerank/reranker.go index cf382c09..bc551dfb 100644 --- a/internal/models/rerank/reranker.go +++ b/internal/models/rerank/reranker.go @@ -4,14 +4,18 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/Tencent/WeKnora/internal/logger" - "github.com/Tencent/WeKnora/internal/models/provider" - // provider.DetectProvider answers from the vendor catalog, which is empty - // until the vendor packages have run their init. Without this import every - // URL detects as "generic" and newReranker below falls through to the - // OpenAI-compatible reranker for LKEAP / Volcengine / Jina / … rows that - // carry no explicit provider id. + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/api/cohererank" + "github.com/Tencent/WeKnora/internal/models/api/dashscoperank" + "github.com/Tencent/WeKnora/internal/models/api/nimrerank" + "github.com/Tencent/WeKnora/internal/models/catalog" + // catalog.Resolve answers from the vendor catalog, which is empty until + // the vendor packages have run their init. Without this import every row + // resolves to the generic vendor: LKEAP and Volcengine would lose their + // signed clients and Aliyun its native protocol. _ "github.com/Tencent/WeKnora/internal/models/vendors" "github.com/Tencent/WeKnora/internal/types" ) @@ -131,45 +135,99 @@ func NewReranker(config *RerankerConfig) (Reranker, error) { return wrapRerankerLangfuse(r, nil) } -// customHeaderSetter 表示支持注入自定义 HTTP header 的 reranker 实现。 -type customHeaderSetter interface { - SetCustomHeaders(map[string]string) -} - +// newReranker resolves the catalog and returns the protocol client for the +// configured model, wrapped in the shared batching and score-scaling layer. +// It mirrors chat.NewRemoteChat: the vendor's facts decide the protocol, the +// URL and the credential, and this function knows no vendor names. func newReranker(config *RerankerConfig) (Reranker, error) { - // Use provider field if set, otherwise detect from URL using provider registry - providerName := provider.ProviderName(config.Provider) - if providerName == "" { - providerName = provider.DetectProvider(config.BaseURL) + if config == nil { + return nil, fmt.Errorf("rerank config is nil") + } + resolved, err := catalog.Resolve(catalog.Ref{ + Provider: config.Provider, + Model: config.ModelName, + BaseURL: config.BaseURL, + ModelType: types.ModelTypeRerank, + Extra: config.ExtraConfig, + }) + if err != nil { + return nil, err + } + if err := validateRerankBaseURL(resolved.BaseURL); err != nil { + return nil, err } - var ( - reranker Reranker - err error - ) - switch providerName { - case provider.ProviderAliyun: - reranker, err = NewAliyunReranker(config) - case provider.ProviderZhipu: - reranker, err = NewZhipuReranker(config) - case provider.ProviderJina: - reranker, err = NewJinaReranker(config) - case provider.ProviderNvidia: - reranker, err = NewNvidiaReranker(config) - case provider.ProviderWeKnoraCloud: - reranker, err = NewWeKnoraCloudReranker(config) - case provider.ProviderLKEAP: - reranker, err = NewLKEAPReranker(config) - case provider.ProviderVolcengine: - reranker, err = NewVolcengineReranker(config) + vendor := resolved.Vendor + creds := catalog.Credentials{APIKey: config.APIKey, AppID: config.AppID, AppSecret: config.AppSecret} + if creds.APIKey == "" { + creds.APIKey = vendor.DefaultAPIKey + } + // A signing vendor with no identity pair would otherwise send unsigned + // requests and fail at the far end, which is a worse error than this one. + if vendor.Auth == catalog.AuthSigned { + if creds.AppID == "" { + return nil, fmt.Errorf("%s rerank: AppID is required", vendor.Name) + } + if creds.AppSecret == "" { + return nil, fmt.Errorf("%s rerank: AppSecret is required", vendor.Name) + } + } + // Vendor headers first so a user header cannot silently replace a vendor + // beta flag, matching chat.NewRemoteChat. + headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders)) + for k, v := range vendor.Headers { + headers[k] = v + } + for k, v := range config.CustomHeaders { + headers[k] = v + } + endpoint := api.Endpoint{ + BaseURL: resolved.BaseURL, + Model: resolved.RemoteModel, + ModelID: config.ModelID, + Auth: vendor.AuthFunc(vendor.API, creds), + Headers: headers, + // Most vendors have always run without a client deadline here and let + // the caller's context govern; the ones that declare a timeout get it. + Client: newRerankHTTPClient( + time.Duration(resolved.Rerank.RequestTimeout) * time.Second, + ), + } + if vendor.Endpoint != nil { + if url, query := vendor.Endpoint(catalog.EndpointRequest{ + BaseURL: resolved.BaseURL, + Model: resolved.RemoteModel, + ModelType: types.ModelTypeRerank, + Extra: config.ExtraConfig, + }); url != "" { + endpoint.URL, endpoint.Query = url, query + } + } + + var client api.Reranker + switch resolved.RerankAPI { + case api.RerankCohere: + client = cohererank.New(cohererank.Config{Endpoint: endpoint, Settings: resolved.Rerank}) + case api.RerankDashScope: + client = dashscoperank.New(dashscoperank.Config{Endpoint: endpoint, Settings: resolved.Rerank}) + case api.RerankNIM: + client = nimrerank.New(nimrerank.Config{Endpoint: endpoint, Settings: resolved.Rerank}) + case api.RerankTencentLKEAP: + client, err = newLKEAPClient(config, resolved) + case api.RerankVolcengineKnowledge: + client, err = newVolcengineClient(config, resolved) default: - reranker, err = NewOpenAIReranker(config) + return nil, fmt.Errorf("unsupported rerank api %q for provider %s", resolved.RerankAPI, vendor.ID) } if err != nil { return nil, err } - if setter, ok := reranker.(customHeaderSetter); ok { - setter.SetCustomHeaders(config.CustomHeaders) - } - return reranker, nil + + return &protocolReranker{ + inner: client, + settings: resolved.Rerank, + endpoint: resolved.BaseURL, + modelName: config.ModelName, + modelID: config.ModelID, + }, nil } diff --git a/internal/models/rerank/sdk_clients_test.go b/internal/models/rerank/sdk_clients_test.go new file mode 100644 index 00000000..49904157 --- /dev/null +++ b/internal/models/rerank/sdk_clients_test.go @@ -0,0 +1,152 @@ +package rerank + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/models/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The two SDK-backed vendors have no protocol package, so their outbound +// shape is not covered by a golden test. These pin what the SDK actually puts +// on the wire, which is what the per-client tests did before the migration. + +const volcengineRerankPath = "/api/knowledge/service/rerank" + +func TestVolcengineClientOutboundShape(t *testing.T) { + withRerankSSRFWhitelist(t, "127.0.0.1") + + var request struct { + Datas []struct { + Query string `json:"query"` + Content *string `json:"content"` + } `json:"datas"` + RerankModel *string `json:"rerank_model"` + RerankInstruction *string `json:"rerank_instruction"` + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, volcengineRerankPath, r.URL.Path) + // The access key identifies the caller in the signature; the secret + // signs it and must never travel. + assert.Contains(t, r.Header.Get("Authorization"), "AKLT-test") + assert.NotContains(t, r.Header.Get("Authorization"), "secret-test") + require.NoError(t, json.NewDecoder(r.Body).Decode(&request)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.91,0.27]}}`)) + })) + defer server.Close() + + client, err := newVolcengineClient( + &RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"}, + &catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"}, + ) + require.NoError(t, err) + + got, err := client.Rerank(context.Background(), "保留对话数据吗", []string{"会保留", "不会保留"}) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, 0, got[0].Index) + assert.Equal(t, "会保留", got[0].Text) + assert.InDelta(t, 0.91, got[0].Score, 1e-4) + assert.Equal(t, 1, got[1].Index) + assert.InDelta(t, 0.27, got[1].Score, 1e-4) + + require.NotNil(t, request.RerankModel) + assert.Equal(t, "doubao-seed-rerank", *request.RerankModel) + require.Len(t, request.Datas, 2) + assert.Equal(t, "保留对话数据吗", request.Datas[0].Query) + require.NotNil(t, request.Datas[0].Content) + assert.Equal(t, "会保留", *request.Datas[0].Content) + require.NotNil(t, request.RerankInstruction) + assert.Contains(t, *request.RerankInstruction, "Document") +} + +func TestVolcengineClientSurfacesAnInBodyErrorCode(t *testing.T) { + withRerankSSRFWhitelist(t, "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"code":1000030,"message":"quota exceeded","data":{}}`)) + })) + defer server.Close() + + client, err := newVolcengineClient( + &RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"}, + &catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"}, + ) + require.NoError(t, err) + + _, err = client.Rerank(context.Background(), "q", []string{"d"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "quota exceeded") +} + +// A score list that does not line up with the documents cannot be mapped back +// onto them, so it is an error rather than a partially filled result. +func TestVolcengineClientRejectsAScoreCountMismatch(t *testing.T) { + withRerankSSRFWhitelist(t, "127.0.0.1") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"code":0,"data":{"scores":[0.5]}}`)) + })) + defer server.Close() + + client, err := newVolcengineClient( + &RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"}, + &catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"}, + ) + require.NoError(t, err) + + _, err = client.Rerank(context.Background(), "q", []string{"d0", "d1"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "score count mismatch") +} + +func TestVolcengineClientRequiresAKAndSK(t *testing.T) { + for _, cfg := range []*RerankerConfig{ + {AppSecret: "sk"}, + {APIKey: "ak"}, + } { + _, err := newVolcengineClient(cfg, &catalog.Resolved{BaseURL: "https://example.invalid"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "access key and secret key are required") + } +} + +func TestLKEAPClientRequiresCredentials(t *testing.T) { + for _, cfg := range []*RerankerConfig{ + {AppSecret: "sk"}, + {APIKey: "id"}, + } { + _, err := newLKEAPClient(cfg, &catalog.Resolved{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "secret_id and secret_key are required") + } +} + +// The secret key has always been accepted from extra_config as well, because +// the model editor stores it there for this vendor. +func TestLKEAPClientTakesTheSecretKeyFromExtraConfig(t *testing.T) { + client, err := newLKEAPClient( + &RerankerConfig{APIKey: "AKIDxxx", ExtraConfig: map[string]string{"secret_key": "sk"}}, + &catalog.Resolved{RemoteModel: "lke-reranker-base"}, + ) + require.NoError(t, err) + require.NotNil(t, client) +} + +func TestLKEAPClientFallsBackToTheDefaultModel(t *testing.T) { + client, err := newLKEAPClient( + &RerankerConfig{APIKey: "AKIDxxx", AppSecret: "sk"}, + &catalog.Resolved{}, + ) + require.NoError(t, err) + inner, ok := client.(*lkeapClient) + require.True(t, ok) + assert.Equal(t, LKEAPDefaultRerankModel, inner.modelName) + assert.False(t, strings.Contains(inner.modelName, " ")) +} diff --git a/internal/models/rerank/transport_security_test.go b/internal/models/rerank/transport_security_test.go index 4aa4d17f..66fd2c7f 100644 --- a/internal/models/rerank/transport_security_test.go +++ b/internal/models/rerank/transport_security_test.go @@ -16,19 +16,19 @@ func withRerankSSRFWhitelist(t *testing.T, raw string) { t.Cleanup(secutils.ResetSSRFWhitelistForTest) } -func TestOpenAIRerankerRejectsInternalBaseURL(t *testing.T) { +func TestRerankerRejectsInternalBaseURL(t *testing.T) { withRerankSSRFWhitelist(t, "") - _, err := NewOpenAIReranker(&RerankerConfig{ + _, err := NewReranker(&RerankerConfig{ BaseURL: "http://169.254.169.254/latest/meta-data/", ModelName: "rerank-test", }) if err == nil { - t.Fatalf("NewOpenAIReranker returned nil error for blocked internal BaseURL") + t.Fatalf("NewReranker returned nil error for blocked internal BaseURL") } } -func TestOpenAIRerankerBlocksRedirectToInternalURL(t *testing.T) { +func TestRerankerBlocksRedirectToInternalURL(t *testing.T) { withRerankSSRFWhitelist(t, "127.0.0.1") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -36,13 +36,13 @@ func TestOpenAIRerankerBlocksRedirectToInternalURL(t *testing.T) { })) defer server.Close() - reranker, err := NewOpenAIReranker(&RerankerConfig{ + reranker, err := NewReranker(&RerankerConfig{ BaseURL: server.URL, ModelName: "rerank-test", APIKey: "sk-test", }) if err != nil { - t.Fatalf("NewOpenAIReranker: %v", err) + t.Fatalf("NewReranker: %v", err) } _, err = reranker.Rerank(t.Context(), "query", []string{"doc"}) diff --git a/internal/models/rerank/volcengine_reranker.go b/internal/models/rerank/volcengine_reranker.go index 2a1b5176..9da65ab2 100644 --- a/internal/models/rerank/volcengine_reranker.go +++ b/internal/models/rerank/volcengine_reranker.go @@ -6,37 +6,33 @@ import ( "strings" "time" - "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/api" + "github.com/Tencent/WeKnora/internal/models/catalog" "github.com/Tencent/WeKnora/internal/models/provider" "github.com/volcengine/vikingdb-go-sdk/knowledge" knowledgemodel "github.com/volcengine/vikingdb-go-sdk/knowledge/model" - "golang.org/x/sync/errgroup" ) const ( + // VolcengineRerankBaseURL is the managed Knowledge Service host. VolcengineRerankBaseURL = provider.VolcengineRerankBaseURL - volcengineRerankPath = "/api/knowledge/service/rerank" volcengineRerankDefaultModel = "doubao-seed-rerank" volcengineRerankDefaultRegion = "cn-beijing" volcengineRerankDefaultInstruction = "Whether the Document answers the Query or matches the content retrieval intent" - volcengineRerankMaxDocuments = 50 - // volcengineRerankMaxConcurrency bounds the number of in-flight batch - // requests when the candidate set exceeds volcengineRerankMaxDocuments, so a - // very large embedding_top_k cannot fan out into an unbounded burst of calls. - volcengineRerankMaxConcurrency = 4 ) -// VolcengineReranker calls the managed Knowledge Service Rerank API with AK/SK signing. -type VolcengineReranker struct { +// volcengineClient calls the managed Knowledge Service rerank through the +// vikingdb SDK, which owns the AK/SK signing. It implements api.Reranker and +// nothing else: the 50-document ceiling and the batch concurrency are +// declared on the vendor and enforced by protocolReranker. +type volcengineClient struct { modelName string instruction string - modelID string - endpoint string client *knowledge.Client } -func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error) { +func newVolcengineClient(config *RerankerConfig, resolved *catalog.Resolved) (api.Reranker, error) { accessKey := strings.TrimSpace(config.APIKey) secretKey := strings.TrimSpace(config.AppSecret) if secretKey == "" && config.ExtraConfig != nil { @@ -46,15 +42,12 @@ func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error) return nil, fmt.Errorf("access key and secret key are required for Volcengine rerank") } - baseURL := strings.TrimRight(strings.TrimSpace(config.BaseURL), "/") + baseURL := resolved.BaseURL if baseURL == "" { baseURL = VolcengineRerankBaseURL } - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - modelName := strings.TrimSpace(config.ModelName) + modelName := strings.TrimSpace(resolved.RemoteModel) if modelName == "" { modelName = volcengineRerankDefaultModel } @@ -80,70 +73,17 @@ func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error) if err != nil { return nil, fmt.Errorf("create Volcengine rerank client: %w", err) } - - return &VolcengineReranker{ - modelName: modelName, - instruction: instruction, - modelID: config.ModelID, - endpoint: baseURL, - client: client, - }, nil + return &volcengineClient{modelName: modelName, instruction: instruction, client: client}, nil } -func (r *VolcengineReranker) Rerank( +// Rerank scores one batch. Every document is paired with the same query and +// instruction, and the reply is a score list in input order. +func (r *volcengineClient) Rerank( ctx context.Context, query string, documents []string, -) ([]RankResult, error) { - if len(documents) == 0 { - return []RankResult{}, nil - } - - // The managed Knowledge Service Rerank API rejects requests carrying more - // than volcengineRerankMaxDocuments items. Upstream callers (chat pipeline, - // agent knowledge search, message search) feed in every retrieval candidate - // and do not cap the count per provider, so a large embedding_top_k or a - // multi-target search can exceed the limit. Each Data item is scored - // independently against the same (query, instruction) pair, so the scores - // are comparable across requests — we can split the documents into limit- - // sized batches, rerank them concurrently, and merge without losing any - // candidate (unlike truncation) or biasing the ranking. - results := make([]RankResult, len(documents)) - g, gctx := errgroup.WithContext(ctx) - g.SetLimit(volcengineRerankMaxConcurrency) - for start := 0; start < len(documents); start += volcengineRerankMaxDocuments { - start := start - end := min(start+volcengineRerankMaxDocuments, len(documents)) - g.Go(func() error { - scores, err := r.rerankBatch(gctx, query, documents[start:end]) - if err != nil { - return err - } - for i, score := range scores { - results[start+i] = RankResult{ - Index: start + i, - Document: DocumentInfo{Text: documents[start+i]}, - RelevanceScore: score, - } - } - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err - } - return results, nil -} - -// rerankBatch scores a single batch of documents (already sized within the API -// limit) and returns the per-document relevance scores in input order. -func (r *VolcengineReranker) rerankBatch( - ctx context.Context, query string, documents []string, -) ([]float64, error) { +) ([]api.RerankResult, error) { data := make([]knowledgemodel.RerankDataItem, len(documents)) for i := range documents { - data[i] = knowledgemodel.RerankDataItem{ - Query: query, - Content: &documents[i], - } + data[i] = knowledgemodel.RerankDataItem{Query: query, Content: &documents[i]} } request := knowledgemodel.RerankRequest{ Datas: data, @@ -151,11 +91,6 @@ func (r *VolcengineReranker) rerankBatch( RerankInstruction: &r.instruction, } - logger.Debugf( - ctx, - "%s", - buildRerankRequestDebug(r.modelName, r.endpoint+volcengineRerankPath, query, documents), - ) response, err := r.client.Rerank(ctx, request) if err != nil { return nil, fmt.Errorf("call Volcengine rerank: %w", err) @@ -169,17 +104,13 @@ func (r *VolcengineReranker) rerankBatch( if len(response.Data.Scores) != len(documents) { return nil, fmt.Errorf( "Volcengine rerank score count mismatch: got %d scores for %d documents", - len(response.Data.Scores), - len(documents), + len(response.Data.Scores), len(documents), ) } - return response.Data.Scores, nil -} -func (r *VolcengineReranker) GetModelName() string { - return r.modelName -} - -func (r *VolcengineReranker) GetModelID() string { - return r.modelID + results := make([]api.RerankResult, len(documents)) + for i, score := range response.Data.Scores { + results[i] = api.RerankResult{Index: i, Score: score, Text: documents[i]} + } + return results, nil } diff --git a/internal/models/rerank/volcengine_reranker_test.go b/internal/models/rerank/volcengine_reranker_test.go deleted file mode 100644 index 43cf6207..00000000 --- a/internal/models/rerank/volcengine_reranker_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package rerank - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestVolcengineReranker_Rerank(t *testing.T) { - withRerankSSRFWhitelist(t, "127.0.0.1") - var request struct { - Datas []struct { - Query string `json:"query"` - Content *string `json:"content"` - } `json:"datas"` - RerankModel *string `json:"rerank_model"` - RerankInstruction *string `json:"rerank_instruction"` - } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, http.MethodPost, r.Method) - assert.Equal(t, volcengineRerankPath, r.URL.Path) - assert.Contains(t, r.Header.Get("Authorization"), "AKLT-test") - assert.NotContains(t, r.Header.Get("Authorization"), "secret-test") - require.NoError(t, json.NewDecoder(r.Body).Decode(&request)) - - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.91,0.27]}}`)) - })) - defer server.Close() - - reranker, err := NewVolcengineReranker(&RerankerConfig{ - APIKey: "AKLT-test", - AppSecret: "secret-test", - BaseURL: server.URL, - ModelName: "doubao-seed-rerank", - ModelID: "volc-rerank", - }) - require.NoError(t, err) - - results, err := reranker.Rerank(t.Context(), "保留对话数据吗", []string{"会保留", "不会保留"}) - require.NoError(t, err) - require.Len(t, results, 2) - assert.Equal(t, 0, results[0].Index) - assert.Equal(t, "会保留", results[0].Document.Text) - assert.InDelta(t, 0.91, results[0].RelevanceScore, 0.0001) - assert.Equal(t, 1, results[1].Index) - assert.Equal(t, "不会保留", results[1].Document.Text) - assert.InDelta(t, 0.27, results[1].RelevanceScore, 0.0001) - - require.NotNil(t, request.RerankModel) - assert.Equal(t, "doubao-seed-rerank", *request.RerankModel) - require.Len(t, request.Datas, 2) - assert.Equal(t, "保留对话数据吗", request.Datas[0].Query) - require.NotNil(t, request.Datas[0].Content) - assert.Equal(t, "会保留", *request.Datas[0].Content) - require.NotNil(t, request.RerankInstruction) - assert.True(t, strings.Contains(*request.RerankInstruction, "Document")) -} - -func TestNewVolcengineReranker_RequiresAKSK(t *testing.T) { - _, err := NewVolcengineReranker(&RerankerConfig{ - APIKey: "ark-api-key-only", - BaseURL: VolcengineRerankBaseURL, - ModelName: "doubao-seed-rerank", - }) - require.Error(t, err) - assert.Contains(t, err.Error(), "access key and secret key") -} - -func newTestVolcengineReranker(t *testing.T, handler http.HandlerFunc) *VolcengineReranker { - t.Helper() - withRerankSSRFWhitelist(t, "127.0.0.1") - server := httptest.NewServer(handler) - t.Cleanup(server.Close) - reranker, err := NewVolcengineReranker(&RerankerConfig{ - APIKey: "AKLT-test", - AppSecret: "secret-test", - BaseURL: server.URL, - ModelName: "doubao-seed-rerank", - }) - require.NoError(t, err) - return reranker -} - -func TestVolcengineReranker_EmptyDocuments(t *testing.T) { - reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) { - t.Fatalf("rerank endpoint should not be called for empty documents") - }) - - results, err := reranker.Rerank(t.Context(), "query", nil) - require.NoError(t, err) - assert.Empty(t, results) -} - -// TestVolcengineReranker_BatchesOverLimit verifies that a candidate set larger -// than the API document limit is split into batches and reranked in full, -// rather than truncated: every document gets a score and no batch exceeds the -// per-request limit. -func TestVolcengineReranker_BatchesOverLimit(t *testing.T) { - var ( - mu sync.Mutex - batchSizes []int - requestCount int - ) - reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) { - var body struct { - Datas []struct { - Content *string `json:"content"` - } `json:"datas"` - } - require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) - - mu.Lock() - requestCount++ - batchSizes = append(batchSizes, len(body.Datas)) - mu.Unlock() - - scores := make([]string, len(body.Datas)) - for i := range scores { - scores[i] = "0.5" - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[` + - strings.Join(scores, ",") + `]}}`)) - }) - - total := volcengineRerankMaxDocuments*2 + 10 - documents := make([]string, total) - for i := range documents { - documents[i] = fmt.Sprintf("doc-%d", i) - } - - results, err := reranker.Rerank(t.Context(), "query", documents) - require.NoError(t, err) - - // All documents reranked, each mapped back to its original index/text. - require.Len(t, results, total) - for i := range results { - assert.Equal(t, i, results[i].Index) - assert.Equal(t, documents[i], results[i].Document.Text) - } - - // Split into ceil(total/limit) batches, none exceeding the API limit. - assert.Equal(t, 3, requestCount) - for _, size := range batchSizes { - assert.LessOrEqual(t, size, volcengineRerankMaxDocuments) - } -} - -func TestVolcengineReranker_APIErrorCode(t *testing.T) { - reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"code":100004,"message":"quota exceeded","data":{}}`)) - }) - - _, err := reranker.Rerank(t.Context(), "query", []string{"a", "b"}) - require.Error(t, err) - assert.Contains(t, err.Error(), "100004") - assert.Contains(t, err.Error(), "quota exceeded") -} - -func TestVolcengineReranker_ScoreCountMismatch(t *testing.T) { - reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.9]}}`)) - }) - - _, err := reranker.Rerank(t.Context(), "query", []string{"a", "b"}) - require.Error(t, err) - assert.Contains(t, err.Error(), "score count mismatch") -} diff --git a/internal/models/rerank/weknoracloud.go b/internal/models/rerank/weknoracloud.go deleted file mode 100644 index 393ae8c2..00000000 --- a/internal/models/rerank/weknoracloud.go +++ /dev/null @@ -1,134 +0,0 @@ -package rerank - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - "github.com/Tencent/WeKnora/internal/models/utils" - "github.com/google/uuid" -) - -const weKnoraCloudRerankPath = "/api/v1/rerank" - -// WeKnoraCloudReranker 实现 rerank.Reranker 接口,对接 WeKnoraCloud /api/v1/rerank -type WeKnoraCloudReranker struct { - modelName string - remoteModelName string - modelID string - appID string - apiKey string - baseURL string - client *http.Client -} - -// NewWeKnoraCloudReranker 构造 WeKnoraCloudReranker -func NewWeKnoraCloudReranker(config *RerankerConfig) (*WeKnoraCloudReranker, error) { - if config.AppID == "" { - return nil, fmt.Errorf("WeKnoraCloud reranker: AppID is required") - } - if config.AppSecret == "" { - return nil, fmt.Errorf("WeKnoraCloud reranker: AppSecret is required") - } - baseURL := strings.TrimRight(config.BaseURL, "/") - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - remoteModelName := "" - if config.ExtraConfig != nil { - remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"]) - } - return &WeKnoraCloudReranker{ - modelName: config.ModelName, - remoteModelName: remoteModelName, - modelID: config.ModelID, - appID: config.AppID, - apiKey: config.AppSecret, - baseURL: baseURL, - client: newRerankHTTPClient(60 * time.Second), - }, nil -} - -type weKnoraCloudRerankRequest struct { - Model string `json:"model"` - Query string `json:"query"` - Documents []string `json:"documents"` -} - -type weKnoraCloudRerankResponse struct { - Results []struct { - Index int `json:"index"` - RelevanceScore float64 `json:"relevance_score"` - Document struct { - Text string `json:"text"` - } `json:"document"` - } `json:"results"` -} - -func (r *WeKnoraCloudReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - reqBody := weKnoraCloudRerankRequest{ - Model: r.effectiveModelName(), - Query: query, - Documents: documents, - } - bodyBytes, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("weknoracloud reranker: marshal: %w", err) - } - - requestID := uuid.New().String() - headers := utils.Sign(r.appID, r.apiKey, requestID, string(bodyBytes)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.baseURL+weKnoraCloudRerankPath, bytes.NewReader(bodyBytes)) - if err != nil { - return nil, fmt.Errorf("weknoracloud reranker: create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - for k, v := range headers { - req.Header.Set(k, v) - } - - resp, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("weknoracloud reranker: do request: %w", err) - } - defer resp.Body.Close() - - respBytes, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("weknoracloud reranker: read response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("weknoracloud reranker: status %d: %s", resp.StatusCode, string(respBytes)) - } - - var rerankResp weKnoraCloudRerankResponse - if err := json.Unmarshal(respBytes, &rerankResp); err != nil { - return nil, fmt.Errorf("weknoracloud reranker: unmarshal: %w", err) - } - - results := make([]RankResult, 0, len(rerankResp.Results)) - for _, item := range rerankResp.Results { - results = append(results, RankResult{ - Index: item.Index, - RelevanceScore: item.RelevanceScore, - Document: DocumentInfo{Text: item.Document.Text}, - }) - } - return results, nil -} - -func (r *WeKnoraCloudReranker) effectiveModelName() string { - if r.remoteModelName != "" { - return r.remoteModelName - } - return r.modelName -} - -func (r *WeKnoraCloudReranker) GetModelName() string { return r.modelName } -func (r *WeKnoraCloudReranker) GetModelID() string { return r.modelID } diff --git a/internal/models/rerank/zhipu_reranker.go b/internal/models/rerank/zhipu_reranker.go deleted file mode 100644 index c1c70468..00000000 --- a/internal/models/rerank/zhipu_reranker.go +++ /dev/null @@ -1,153 +0,0 @@ -package rerank - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/Tencent/WeKnora/internal/logger" - secutils "github.com/Tencent/WeKnora/internal/utils" -) - -// ZhipuReranker implements a reranking system based on Zhipu AI models -type ZhipuReranker struct { - modelName string // Name of the model used for reranking - modelID string // Unique identifier of the model - apiKey string // API key for authentication - baseURL string // Base URL for API requests - client *http.Client // HTTP client for making API requests - customHeaders map[string]string -} - -// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。 -func (r *ZhipuReranker) SetCustomHeaders(headers map[string]string) { - r.customHeaders = headers -} - -// ZhipuRerankRequest represents a request to rerank documents using Zhipu AI API -type ZhipuRerankRequest struct { - Model string `json:"model"` // Model to use for reranking - Query string `json:"query"` // Query text to compare documents against - Documents []string `json:"documents"` // List of document texts to rerank - TopN int `json:"top_n,omitempty"` // Number of top results to return (0 = all) - ReturnDocuments bool `json:"return_documents,omitempty"` // Whether to return documents in response - ReturnRawScores bool `json:"return_raw_scores,omitempty"` // Whether to return raw scores -} - -// ZhipuRerankResponse represents the response from Zhipu AI reranking request -type ZhipuRerankResponse struct { - RequestID string `json:"request_id"` // Request ID from client or platform - ID string `json:"id"` // Task order ID from Zhipu platform - Results []ZhipuRankResult `json:"results"` // Ranked results with relevance scores - Usage ZhipuUsage `json:"usage"` // Token usage information -} - -// ZhipuRankResult represents a single reranking result from Zhipu AI -type ZhipuRankResult struct { - Index int `json:"index"` // Original index of the document - RelevanceScore float64 `json:"relevance_score"` // Relevance score - Document string `json:"document,omitempty"` // Document text (optional) -} - -// ZhipuUsage contains information about token usage in the Zhipu API request -type ZhipuUsage struct { - TotalTokens int `json:"total_tokens"` // Total tokens consumed - PromptTokens int `json:"prompt_tokens"` // Prompt tokens -} - -// NewZhipuReranker creates a new instance of Zhipu reranker with the provided configuration -func NewZhipuReranker(config *RerankerConfig) (*ZhipuReranker, error) { - apiKey := config.APIKey - baseURL := "https://open.bigmodel.cn/api/paas/v4/rerank" - if url := config.BaseURL; url != "" { - baseURL = url - } - if err := validateRerankBaseURL(baseURL); err != nil { - return nil, err - } - - return &ZhipuReranker{ - modelName: config.ModelName, - modelID: config.ModelID, - apiKey: apiKey, - baseURL: baseURL, - client: newRerankHTTPClient(0), - }, nil -} - -// Rerank performs document reranking based on relevance to the query using Zhipu AI API -func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { - // Build the request body - requestBody := &ZhipuRerankRequest{ - Model: r.modelName, - Query: query, - Documents: documents, - TopN: 0, // Return all documents - ReturnDocuments: true, - ReturnRawScores: false, - } - - jsonData, err := json.Marshal(requestBody) - if err != nil { - return nil, fmt.Errorf("marshal request body: %w", err) - } - - // Send the request - req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) - secutils.ApplyCustomHeaders(req, r.customHeaders) - - logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents)) - - resp, err := r.client.Do(req) - if err != nil { - return nil, fmt.Errorf("do request: %w", err) - } - defer resp.Body.Close() - - // Read the response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("zhipu rerank API error: Http Status: %s, Body: %s", resp.Status, string(body)) - } - - var response ZhipuRerankResponse - if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("unmarshal response: %w", err) - } - - // Convert Zhipu results to standard RankResult format - results := make([]RankResult, len(response.Results)) - for i, zhipuResult := range response.Results { - results[i] = RankResult{ - Index: zhipuResult.Index, - Document: DocumentInfo{ - Text: zhipuResult.Document, - }, - RelevanceScore: zhipuResult.RelevanceScore, - } - } - - return results, nil -} - -// GetModelName returns the name of the reranking model -func (r *ZhipuReranker) GetModelName() string { - return r.modelName -} - -// GetModelID returns the unique identifier of the reranking model -func (r *ZhipuReranker) GetModelID() string { - return r.modelID -} diff --git a/internal/models/vendors/aliyun/models.json b/internal/models/vendors/aliyun/models.json index 0e8d085b..efb3f080 100644 --- a/internal/models/vendors/aliyun/models.json +++ b/internal/models/vendors/aliyun/models.json @@ -47,7 +47,7 @@ {"id": "text-embedding-v4", "name": "Tongyi text-embedding-v4", "type": "Embedding", "dimension": 1024, "source": "https://help.aliyun.com/zh/model-studio/embedding"}, {"id": "text-embedding-v3", "name": "Tongyi text-embedding-v3", "type": "Embedding", "dimension": 1024, "source": "https://help.aliyun.com/zh/model-studio/embedding"}, {"id": "tongyi-embedding-vision-plus", "name": "Tongyi Embedding Vision Plus", "type": "Embedding", "dimension": 1152, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api"}, - {"id": "qwen3-rerank", "name": "Qwen3 Rerank", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}, + {"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}, {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"} ] } diff --git a/internal/models/vendors/aliyun/vendor.go b/internal/models/vendors/aliyun/vendor.go index c23360f4..0421dba2 100644 --- a/internal/models/vendors/aliyun/vendor.go +++ b/internal/models/vendors/aliyun/vendor.go @@ -58,12 +58,16 @@ // - rerank is a separate DashScope-native endpoint // (/api/v1/services/rerank/text-rerank/text-rerank). // -// unverified: the docs place qwen3-rerank on a different path than every -// other rerank model (/compatible-api/v1/reranks instead of the native -// text-rerank path this package defaults to), and WeKnora's DashScope rerank -// client only speaks the native request shape. The single RerankBaseURL is -// kept; qwen3-rerank needs an operator-supplied base URL until the client -// learns the second shape +// qwen3-rerank is a second, incompatible rerank protocol on the same vendor. +// The text-rerank page puts it on /compatible-api/v1/reranks and states +// outright that "两种接口的请求体结构和响应格式不同": its request is flat +// (query / documents at the top level, no input/parameters wrapper) and its +// response carries `results` at the top level with no `output` object. This +// package implements only the native shape that gte-rerank-v2 and +// qwen3.7-text-rerank use, so the entry is marked deprecated: it stays +// resolvable for a row that already names it, but the picker no longer offers +// a model that would be sent to the wrong path and decoded with the wrong +// shape. Serving it needs a fourth rerank protocol package // (https://help.aliyun.com/zh/model-studio/text-rerank-api). // // unverified: no page states whether `prompt_cache_key` is accepted, so the @@ -108,7 +112,7 @@ func init() { ID: ID, Name: "Alibaba Cloud DashScope", Names: map[string]string{"zh-CN": "阿里云 DashScope"}, - Description: "qwen-plus, qwen3.8-max, deepseek-v4-pro, text-embedding-v4, qwen3-rerank, etc.", + Description: "qwen-plus, qwen3.8-max, deepseek-v4-pro, text-embedding-v4, gte-rerank-v2, etc.", Website: "https://bailian.console.aliyun.com", Icon: icon, API: api.APIOpenAICompletions, @@ -128,7 +132,16 @@ func init() { types.ModelTypeRerank, types.ModelTypeVLLM, }, + RerankAPI: api.RerankDashScope, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + SendReturnDocs: catalog.Ptr(true), + // 500 documents per request for the native text-rerank models. + // The query (4,000 tokens) and per-document limits are stated in + // tokens, which a rune count cannot express, so they are not + // declared. + MaxDocuments: catalog.Ptr(500), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ // Explicit although it matches the protocol default, because // this vendor's own parameter table deprecates the other diff --git a/internal/models/vendors/azure_openai/vendor.go b/internal/models/vendors/azure_openai/vendor.go index b903d5af..566c3a7f 100644 --- a/internal/models/vendors/azure_openai/vendor.go +++ b/internal/models/vendors/azure_openai/vendor.go @@ -148,6 +148,9 @@ func init() { // api-version. A value switches back to the dated // /openai/deployments/... path. Placeholder: "leave empty for /openai/v1, or e.g. 2025-04-01-preview", + Placeholders: map[string]string{ + "zh-CN": "留空走 /openai/v1,或填如 2025-04-01-preview", + }, }, }, Compat: catalog.VendorCompat{ diff --git a/internal/models/vendors/generic/vendor.go b/internal/models/vendors/generic/vendor.go index e6a881c6..41c7d961 100644 --- a/internal/models/vendors/generic/vendor.go +++ b/internal/models/vendors/generic/vendor.go @@ -71,7 +71,49 @@ func init() { types.ModelTypeVLLM, types.ModelTypeASR, }, + ExtraFields: []catalog.ExtraField{{ + Key: catalog.ExtraScoreScale, + Label: "Rerank score scale", + Labels: map[string]string{"zh-CN": "Rerank 分数标度"}, + Type: "select", + Options: []catalog.ExtraFieldOption{ + { + Label: "Unbounded score (Qwen3-Reranker class)", + Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"}, + Value: "logit", + }, + { + Label: "0..1 relevance (BGE class)", + Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"}, + Value: "probability", + }, + }, + Placeholder: "match the reranker actually deployed behind this endpoint", + Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"}, + // A gateway serves whatever reranker was deployed behind it, and + // the two families disagree. The vendor default follows the + // documentation; the operator knows which model is actually there. + Required: false, + ModelTypes: []types.ModelType{types.ModelTypeRerank}, + }, { + Key: catalog.ExtraTruncatePromptTokens, + Label: "Rerank prompt truncation (vLLM)", + Labels: map[string]string{"zh-CN": "Rerank 提示词截断长度(vLLM)"}, + Type: "number", + // Off unless the operator asks: a runtime that does not implement + // the extension rejects the unknown field. + Required: false, + Placeholder: "empty unless the backend rejects long documents", + Placeholders: map[string]string{"zh-CN": "留空,除非后端因文档过长而报错"}, + ModelTypes: []types.ModelType{types.ModelTypeRerank}, + }}, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // Any OpenAI-compatible endpoint an operator points here is most + // often a vLLM or SGLang server, which is where + // truncate_prompt_tokens comes from. + AcceptsTruncatePromptTokens: catalog.Ptr(true), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ MaxTokensField: catalog.Ptr("max_tokens"), ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs), diff --git a/internal/models/vendors/gpustack/vendor.go b/internal/models/vendors/gpustack/vendor.go index 1d3ea2bf..cae8ee7c 100644 --- a/internal/models/vendors/gpustack/vendor.go +++ b/internal/models/vendors/gpustack/vendor.go @@ -81,7 +81,55 @@ func init() { types.ModelTypeVLLM, types.ModelTypeASR, }, + ExtraFields: []catalog.ExtraField{{ + Key: catalog.ExtraScoreScale, + Label: "Rerank score scale", + Labels: map[string]string{"zh-CN": "Rerank 分数标度"}, + Type: "select", + Options: []catalog.ExtraFieldOption{ + { + Label: "Unbounded score (Qwen3-Reranker class)", + Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"}, + Value: "logit", + }, + { + Label: "0..1 relevance (BGE class)", + Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"}, + Value: "probability", + }, + }, + Placeholder: "match the reranker actually deployed behind this endpoint", + Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"}, + // A gateway serves whatever reranker was deployed behind it, and + // the two families disagree. The vendor default follows the + // documentation; the operator knows which model is actually there. + Required: false, + ModelTypes: []types.ModelType{types.ModelTypeRerank}, + }, { + Key: catalog.ExtraTruncatePromptTokens, + Label: "Rerank prompt truncation (vLLM)", + Labels: map[string]string{"zh-CN": "Rerank 提示词截断长度(vLLM)"}, + Type: "number", + // Off unless the operator asks: a runtime that does not implement + // the extension rejects the unknown field. + Required: false, + Placeholder: "empty unless the backend rejects long documents", + Placeholders: map[string]string{"zh-CN": "留空,除非后端因文档过长而报错"}, + ModelTypes: []types.ModelType{types.ModelTypeRerank}, + }}, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // GPUStack's built-in backends are vLLM, SGLang, Ascend MindIE and + // VoxBox, so the vLLM rerank extension reaches the model. + AcceptsTruncatePromptTokens: catalog.Ptr(true), + // The field is spelled relevance_score like the Cohere shape, but + // it is not a probability: the rerank API page's own example + // returns 1.951932668685913, -3.7347371578216553 and + // -6.157620906829834. Read as a probability, every negative score + // falls below a relevance threshold tuned for 0..1. + // https://docs.gpustack.ai/2.0/user-guide/rerank-api/ + ScoreScale: catalog.Ptr(api.ScoreLogit), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ MaxTokensField: catalog.Ptr("max_tokens"), ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs), diff --git a/internal/models/vendors/jina/vendor.go b/internal/models/vendors/jina/vendor.go index 57269c35..d09eaf37 100644 --- a/internal/models/vendors/jina/vendor.go +++ b/internal/models/vendors/jina/vendor.go @@ -63,6 +63,14 @@ func init() { types.ModelTypeEmbedding, types.ModelTypeRerank, }, + Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // return_documents echoes the text back. Results are matched + // by index, so this is not needed to map them; it is kept + // because it is what this vendor has always been sent. + SendReturnDocs: catalog.Ptr(true), + }, + }, Models: catalog.MustParseModels(modelsJSON), }) } diff --git a/internal/models/vendors/litellm/vendor.go b/internal/models/vendors/litellm/vendor.go index 0d40bf8f..3e40ef00 100644 --- a/internal/models/vendors/litellm/vendor.go +++ b/internal/models/vendors/litellm/vendor.go @@ -48,6 +48,10 @@ const ID = "litellm" // BaseURL is a placeholder the operator replaces with a reachable proxy. const BaseURL = "http://your_litellm_proxy/v1" +// RerankBaseURL is the proxy root. LiteLLM documents rerank at /rerank +// rather than /v1/rerank, so it does not hang off the chat base URL. +const RerankBaseURL = "http://your_litellm_proxy" + func init() { catalog.Register(&catalog.Vendor{ ID: ID, @@ -65,13 +69,40 @@ func init() { DefaultBaseURLs: map[types.ModelType]string{ types.ModelTypeKnowledgeQA: BaseURL, types.ModelTypeEmbedding: BaseURL, + types.ModelTypeRerank: RerankBaseURL, types.ModelTypeVLLM: BaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, + types.ModelTypeRerank, types.ModelTypeVLLM, }, + ExtraFields: []catalog.ExtraField{{ + Key: catalog.ExtraScoreScale, + Label: "Rerank score scale", + Labels: map[string]string{"zh-CN": "Rerank 分数标度"}, + Type: "select", + Options: []catalog.ExtraFieldOption{ + { + Label: "Unbounded score (Qwen3-Reranker class)", + Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"}, + Value: "logit", + }, + { + Label: "0..1 relevance (BGE class)", + Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"}, + Value: "probability", + }, + }, + Placeholder: "match the reranker actually deployed behind this endpoint", + Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"}, + // A gateway serves whatever reranker was deployed behind it, and + // the two families disagree. The vendor default follows the + // documentation; the operator knows which model is actually there. + Required: false, + ModelTypes: []types.ModelType{types.ModelTypeRerank}, + }}, Compat: catalog.VendorCompat{ OpenAICompletions: catalog.OpenAICompletionsCompat{ ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatOpenAI), diff --git a/internal/models/vendors/lkeap/vendor.go b/internal/models/vendors/lkeap/vendor.go index a3a28995..4bce1323 100644 --- a/internal/models/vendors/lkeap/vendor.go +++ b/internal/models/vendors/lkeap/vendor.go @@ -111,8 +111,11 @@ func init() { Type: "password", Required: true, Placeholder: "Tencent Cloud SecretKey (the API Key field holds the SecretId)", - ModelTypes: rerankOnly, - Secret: true, + Placeholders: map[string]string{ + "zh-CN": "腾讯云 SecretKey(API Key 那一栏填的是 SecretId)", + }, + ModelTypes: rerankOnly, + Secret: true, }, { // RunRerank is only published in ap-beijing and ap-guangzhou. @@ -129,7 +132,19 @@ func init() { ModelTypes: rerankOnly, }, }, + RerankAPI: api.RerankTencentLKEAP, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // RunRerank takes at most 60 documents, and Query plus Docs + // together at most 2000 characters. + MaxDocuments: catalog.Ptr(60), + MaxRequestChars: catalog.Ptr(2000), + // Batches went out one at a time before the shared batching + // layer existed. A 2000-character budget splits a large + // candidate set into many requests, so the default fan-out of + // four would be a new burst against RunRerank. + MaxConcurrency: catalog.Ptr(1), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ MaxTokensField: catalog.Ptr("max_tokens"), ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType), diff --git a/internal/models/vendors/novita/vendor.go b/internal/models/vendors/novita/vendor.go index 5fb86a28..0fee0ecd 100644 --- a/internal/models/vendors/novita/vendor.go +++ b/internal/models/vendors/novita/vendor.go @@ -1,8 +1,14 @@ // Package novita registers the Novita AI gateway. // // Facts (https://docs.novita.ai — novita.ai/docs/* redirects there): -// - chat, embedding and VLM share https://api.novita.ai/openai/v1; -// /v1/rerank, /v1/completions, /v1/batches and /v1/files sit alongside; +// - chat, embedding, VLM and rerank share https://api.novita.ai/openai/v1; +// /v1/completions, /v1/batches and /v1/files sit alongside; +// - rerank is the Cohere shape — {model, query, documents, top_n} answering +// results[].{index, relevance_score, document.text} — so it needs no +// vendor settings beyond the type being open +// (https://docs.novita.ai/api-reference/model-apis-llm-create-rerank). +// No ceiling on the document count is documented, and the reference +// lists no model ids, so none are shipped: the operator names the model; // - output cap is `max_tokens`, documented as required; // `max_completion_tokens` is never mentioned; // - the thinking switch is a **top-level** `enable_thinking` boolean @@ -59,11 +65,13 @@ func init() { DefaultBaseURLs: map[types.ModelType]string{ types.ModelTypeKnowledgeQA: BaseURL, types.ModelTypeEmbedding: BaseURL, + types.ModelTypeRerank: BaseURL, types.ModelTypeVLLM: BaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, + types.ModelTypeRerank, types.ModelTypeVLLM, }, Compat: catalog.VendorCompat{ diff --git a/internal/models/vendors/nvidia/vendor.go b/internal/models/vendors/nvidia/vendor.go index 06d1ff59..b940562b 100644 --- a/internal/models/vendors/nvidia/vendor.go +++ b/internal/models/vendors/nvidia/vendor.go @@ -98,7 +98,17 @@ func init() { types.ModelTypeRerank, types.ModelTypeVLLM, }, + RerankAPI: api.RerankNIM, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // rankings[].logit is unbounded and routinely negative, so the + // caller must be told this is not a 0..1 relevance score. + ScoreScale: catalog.Ptr(api.ScoreLogit), + // truncate defaults to NONE upstream, which fails the request on + // an over-long passage instead of cutting it. + Truncate: catalog.Ptr("END"), + MaxDocuments: catalog.Ptr(512), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ MaxTokensField: catalog.Ptr("max_tokens"), ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs), diff --git a/internal/models/vendors/openrouter/models.json b/internal/models/vendors/openrouter/models.json index 7360d5dd..a76f2d05 100644 --- a/internal/models/vendors/openrouter/models.json +++ b/internal/models/vendors/openrouter/models.json @@ -18,6 +18,7 @@ {"id": "minimax/minimax-m3", "name": "MiniMax M3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 512000, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.06}}, {"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}}, {"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"}, - {"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"} + {"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"}, + {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"} ] } diff --git a/internal/models/vendors/openrouter/vendor.go b/internal/models/vendors/openrouter/vendor.go index c01abfb1..fddf72c1 100644 --- a/internal/models/vendors/openrouter/vendor.go +++ b/internal/models/vendors/openrouter/vendor.go @@ -67,11 +67,13 @@ func init() { DefaultBaseURLs: map[types.ModelType]string{ types.ModelTypeKnowledgeQA: BaseURL, types.ModelTypeEmbedding: BaseURL, + types.ModelTypeRerank: BaseURL, types.ModelTypeVLLM: BaseURL, }, ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, types.ModelTypeEmbedding, + types.ModelTypeRerank, types.ModelTypeVLLM, }, Compat: catalog.VendorCompat{ diff --git a/internal/models/vendors/qianfan/vendor.go b/internal/models/vendors/qianfan/vendor.go index 316ef2e9..ee0c5588 100644 --- a/internal/models/vendors/qianfan/vendor.go +++ b/internal/models/vendors/qianfan/vendor.go @@ -101,6 +101,13 @@ func init() { types.ModelTypeVLLM, }, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // "文本数量不超过64"; query "长度不超过1600个字符"; each document + // "长度不超过4096个字符". + MaxDocuments: catalog.Ptr(64), + MaxQueryChars: catalog.Ptr(1600), + MaxDocumentChars: catalog.Ptr(4096), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ // Both spellings are accepted; max_completion_tokens is the // one that also covers the thinking chain. diff --git a/internal/models/vendors/vendors_test.go b/internal/models/vendors/vendors_test.go index 6aee47ea..e7e5ace9 100644 --- a/internal/models/vendors/vendors_test.go +++ b/internal/models/vendors/vendors_test.go @@ -90,6 +90,16 @@ func TestEveryCatalogEntryResolves(t *testing.T) { modelType = types.ModelTypeKnowledgeQA } r, err := catalog.Resolve(catalog.Ref{Provider: v.ID, Model: name, ModelType: modelType}) + // An entry may declare that this build cannot serve it — a vendor + // whose second rerank dialect has no protocol package. Refusing is + // the point: the alternative is a request shaped for the wrong + // protocol. Such an entry must refuse, and must do it with a reason. + if bytes.Contains(m.Compat, []byte("unsupported_reason")) { + if err == nil { + t.Errorf("%s/%s: declares unsupported_reason but still resolves", id, name) + } + continue + } if err != nil { t.Errorf("%s/%s: resolve: %v", id, name, err) continue diff --git a/internal/models/vendors/volcengine/vendor.go b/internal/models/vendors/volcengine/vendor.go index aae1b929..f878bcee 100644 --- a/internal/models/vendors/volcengine/vendor.go +++ b/internal/models/vendors/volcengine/vendor.go @@ -150,8 +150,11 @@ func init() { Type: "password", Required: true, Placeholder: "Volcengine IAM secret key (the API Key field holds the access key)", - ModelTypes: rerankOnly, - Secret: true, + Placeholders: map[string]string{ + "zh-CN": "火山引擎 IAM Secret Access Key(API Key 那一栏填的是 Access Key ID)", + }, + ModelTypes: rerankOnly, + Secret: true, }, { Key: "region", @@ -169,10 +172,18 @@ func init() { Type: "string", Default: "Whether the Document answers the Query or matches the content retrieval intent", Placeholder: "Instruction passed to the rerank model", - ModelTypes: rerankOnly, + Placeholders: map[string]string{ + "zh-CN": "传给重排模型的 instruction", + }, + ModelTypes: rerankOnly, }, }, + RerankAPI: api.RerankVolcengineKnowledge, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + MaxDocuments: catalog.Ptr(50), + MaxConcurrency: catalog.Ptr(4), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType), SupportsReasoningEffort: catalog.Ptr(true), diff --git a/internal/models/vendors/weknoracloud/vendor.go b/internal/models/vendors/weknoracloud/vendor.go index b2e3628d..7895da54 100644 --- a/internal/models/vendors/weknoracloud/vendor.go +++ b/internal/models/vendors/weknoracloud/vendor.go @@ -90,6 +90,13 @@ func init() { types.ModelTypeVLLM, }, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + Path: catalog.Ptr("/api/v1/rerank"), + // The only rerank vendor that has ever had a client deadline + // here. Without it a hung endpoint holds the retrieval stage + // for as long as the caller's context allows. + RequestTimeout: catalog.Ptr(60), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ SupportsMultiContent: catalog.Ptr(false), PromptCacheAccounting: catalog.Ptr(false), diff --git a/internal/models/vendors/zhipu/vendor.go b/internal/models/vendors/zhipu/vendor.go index 24086d18..99bafb21 100644 --- a/internal/models/vendors/zhipu/vendor.go +++ b/internal/models/vendors/zhipu/vendor.go @@ -109,6 +109,14 @@ func init() { types.ModelTypeVLLM, }, Compat: catalog.VendorCompat{ + Rerank: catalog.RerankCompat{ + // The reference gives 最大长度为 4096 字符 for the query and for each + // document, and caps documents at 128 per request. + SendReturnDocs: catalog.Ptr(true), + MaxDocuments: catalog.Ptr(128), + MaxQueryChars: catalog.Ptr(4096), + MaxDocumentChars: catalog.Ptr(4096), + }, OpenAICompletions: catalog.OpenAICompletionsCompat{ MaxTokensField: catalog.Ptr("max_tokens"), ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),