refactor(rerank): move the wire formats into protocol packages, and correct the vendor facts (#3488)
把 8 个手写 rerank client 收敛成 3 个协议包(internal/models/api/{cohererank,dashscoperank,nimrerank}),每个带一份从厂商文档逐字粘来的 golden fixture。LKEAP 和火山保留官方 SDK,但只实现 api.Reranker。rerank.NewReranker 改成 catalog.Resolve + 按协议分发,函数体里没有厂商名。生产代码净 -147 行。
三件原来每个 client 各管各、现在由共用层统一的事:分批(按厂商声明的上限,智谱 128 / NIM 512 等此前无人执行)、分数标度(NIM 与 GPUStack 返回 logit,其余为 0~1 概率,统一换算后再与 RerankThreshold 比较)、NIM 的 truncate(默认 NONE 会让超长 passage 整请求失败,此前从未发送)。
对照官方文档订正四处厂商事实:GPUStack 分数标度、千帆三项上限、阿里 500 条上限、qwen3-rerank 是第二种协议(隐藏并在构造期拒绝)。Novita / OpenRouter / LiteLLM 三家官方有 rerank 但未开类型,本次开放,零协议代码。
运行时行为变更:NVIDIA 起发 truncate: END;GPUStack 分数按 logit 换算(可按行覆盖);truncate_prompt_tokens 收窄为厂商声明制;qwen3-rerank 存量行从静默打错变为构造期报错。详见 PR 描述。
关联 #3486。
main
parent
9aa63c866f
commit
d6c7665cb5
|
|
@ -584,6 +584,7 @@ export function fabriTag(request: FabriTagRequest): Promise<FabriTagResponse> {
|
|||
// Mirrors internal/models/catalog.ExtraField.
|
||||
export interface ModelProviderExtraFieldOption {
|
||||
label: string;
|
||||
labels?: Record<string, string>;
|
||||
value: string;
|
||||
}
|
||||
|
||||
|
|
@ -613,6 +614,7 @@ export interface ModelProviderExtraField {
|
|||
required?: boolean;
|
||||
default?: string;
|
||||
placeholder?: string;
|
||||
placeholders?: Record<string, string>;
|
||||
options?: ModelProviderExtraFieldOption[];
|
||||
// Backend model types ("KnowledgeQA", "Embedding", "Rerank", "VLLM", "ASR"); empty = all.
|
||||
model_types?: string[];
|
||||
|
|
|
|||
|
|
@ -358,22 +358,24 @@
|
|||
<div v-if="field.type === 'boolean'" class="vision-toggle">
|
||||
<t-switch :model-value="extraConfigBool(field.key)"
|
||||
@update:model-value="(v: boolean) => setExtraConfig(field.key, v ? 'true' : 'false')" />
|
||||
<span v-if="field.placeholder" class="form-desc form-desc--inline">{{ field.placeholder }}</span>
|
||||
<span v-if="extraFieldDisplayPlaceholder(field)" class="form-desc form-desc--inline">{{ extraFieldDisplayPlaceholder(field) }}</span>
|
||||
</div>
|
||||
<t-select v-else-if="field.type === 'select'" :model-value="formData.extraConfig[field.key] || ''"
|
||||
:placeholder="field.placeholder || ''" clearable
|
||||
:placeholder="extraFieldDisplayPlaceholder(field)" clearable
|
||||
@update:model-value="(v: string) => setExtraConfig(field.key, v)">
|
||||
<t-option v-for="opt in (field.options || [])" :key="opt.value" :value="opt.value" :label="opt.label || opt.value" />
|
||||
<t-option v-for="opt in (field.options || [])" :key="opt.value" :value="opt.value"
|
||||
:label="extraFieldDisplayOptionLabel(opt)" />
|
||||
</t-select>
|
||||
<t-input v-else-if="field.type === 'number'" :model-value="formData.extraConfig[field.key] || ''"
|
||||
type="number" :placeholder="field.placeholder || ''"
|
||||
type="number" :placeholder="extraFieldDisplayPlaceholder(field)"
|
||||
@update:model-value="(v: string | number) => setExtraConfig(field.key, String(v ?? ''))" />
|
||||
<t-input v-else-if="field.type === 'password'" :model-value="formData.extraConfig[field.key] || ''"
|
||||
type="password" :placeholder="field.placeholder || ''" autocomplete="off" spellcheck="false"
|
||||
type="password" :placeholder="extraFieldDisplayPlaceholder(field)" autocomplete="off" spellcheck="false"
|
||||
@update:model-value="(v: string) => setExtraConfig(field.key, v)">
|
||||
<template #prefix-icon><t-icon name="lock-on" /></template>
|
||||
</t-input>
|
||||
<t-input v-else :model-value="formData.extraConfig[field.key] || ''" :placeholder="field.placeholder || ''"
|
||||
<t-input v-else :model-value="formData.extraConfig[field.key] || ''"
|
||||
:placeholder="extraFieldDisplayPlaceholder(field)"
|
||||
@update:model-value="(v: string) => setExtraConfig(field.key, v)" />
|
||||
</div>
|
||||
|
||||
|
|
@ -599,7 +601,8 @@ import { MessagePlugin, DialogPlugin } from 'tdesign-vue-next'
|
|||
import {
|
||||
checkOllamaModels, checkRemoteModel, testEmbeddingModel, checkRerankModel, checkASRModel, listOllamaModels,
|
||||
downloadOllamaModel, getDownloadProgress, checkOllamaStatus, resolveModelCatalog,
|
||||
type OllamaModelInfo, type ModelProviderOption, type ModelProviderExtraField, type ModelCatalogEntry,
|
||||
type OllamaModelInfo, type ModelProviderOption, type ModelProviderExtraField,
|
||||
type ModelProviderExtraFieldOption, type ModelCatalogEntry,
|
||||
type ResolvedModelCatalog,
|
||||
} from '@/api/initialization'
|
||||
import {
|
||||
|
|
@ -615,6 +618,8 @@ import { useModelProvidersStore } from '@/stores/modelProviders'
|
|||
import {
|
||||
credentialLabelForModelType,
|
||||
extraFieldLabel,
|
||||
extraFieldOptionLabel,
|
||||
extraFieldPlaceholder,
|
||||
extraFieldsForModelType,
|
||||
pickLocalized,
|
||||
providerDescription,
|
||||
|
|
@ -775,6 +780,10 @@ const matchTriggerWidth = (triggerElement: HTMLElement) => ({
|
|||
})
|
||||
|
||||
const extraFieldDisplayLabel = (field: ModelProviderExtraField) => extraFieldLabel(field, currentLocale.value)
|
||||
const extraFieldDisplayPlaceholder = (field: ModelProviderExtraField) =>
|
||||
extraFieldPlaceholder(field, currentLocale.value)
|
||||
const extraFieldDisplayOptionLabel = (option: ModelProviderExtraFieldOption) =>
|
||||
extraFieldOptionLabel(option, currentLocale.value)
|
||||
|
||||
/**
|
||||
* Vendors whose API is not a bearer-token API name their first credential
|
||||
|
|
|
|||
|
|
@ -812,3 +812,66 @@ test('a vendor with no catalog falls back to a plain text field', async () => {
|
|||
} finally { f2.close() }
|
||||
} finally { f.close() }
|
||||
})
|
||||
|
||||
// --- Vendor strings reach the form in the reader's language ---------------
|
||||
|
||||
// The editor renders whatever the catalog declares, so every operator-facing
|
||||
// string a vendor ships has to go through the locale resolvers. Binding a raw
|
||||
// `field.placeholder` or `opt.label` is silent: the form still works, it is
|
||||
// just English in a Chinese product, and a test that only checks the catalog
|
||||
// for a zh-CN entry passes while the editor ignores it. This reads the
|
||||
// template instead.
|
||||
test('every vendor-declared string in the form goes through a locale resolver', () => {
|
||||
const template = descriptor.template?.content ?? ''
|
||||
assert.ok(template.length > 0, 'the editor template should have been parsed')
|
||||
|
||||
// `field` is the extra-field loop variable, so any read of its placeholder
|
||||
// is a vendor string rendered without resolving the locale. One input
|
||||
// branch per field type plus the secret field means it is easy to bind four
|
||||
// and miss the fifth, which is exactly what happened.
|
||||
for (const pattern of [
|
||||
/:placeholder="field\.placeholder/,
|
||||
/\{\{\s*field\.placeholder\s*\}\}/,
|
||||
/v-if="field\.placeholder"/,
|
||||
]) {
|
||||
assert.equal(
|
||||
pattern.test(template), false,
|
||||
`template reads field.placeholder directly (${pattern}); use extraFieldDisplayPlaceholder`,
|
||||
)
|
||||
}
|
||||
|
||||
// Options: scope to the extra-field select, because `opt` is also the loop
|
||||
// variable of the model-type list and the catalog model list, whose labels
|
||||
// come from i18n and from the catalog's own names.
|
||||
const optionLoop = /v-for="opt in \(field\.options \|\| \[\]\)"[^>]*/.exec(template)
|
||||
assert.ok(optionLoop, 'the extra-field select should loop over field.options')
|
||||
assert.match(
|
||||
optionLoop[0], /:label="extraFieldDisplayOptionLabel\(opt\)"/,
|
||||
'extra-field option labels must resolve the locale',
|
||||
)
|
||||
|
||||
// And the placeholder resolver is referenced, so the checks above cannot be
|
||||
// satisfied by dropping the placeholders altogether.
|
||||
assert.ok(template.includes('extraFieldDisplayPlaceholder('), 'placeholders must use the resolver')
|
||||
})
|
||||
|
||||
test('extraFieldDisplayPlaceholder resolves the vendor placeholder for the active locale', async () => {
|
||||
const f = await fixture({ type: 'rerank', providers: catalogProviders })
|
||||
try {
|
||||
const field = {
|
||||
key: 'score_scale',
|
||||
label: 'Rerank score scale',
|
||||
type: 'select',
|
||||
placeholder: 'match the reranker actually deployed behind this endpoint',
|
||||
placeholders: { 'zh-CN': '按这个端点后面实际部署的重排模型选择' },
|
||||
}
|
||||
const rendered = f.vm.extraFieldDisplayPlaceholder(field)
|
||||
assert.ok(
|
||||
rendered === field.placeholder || rendered === field.placeholders['zh-CN'],
|
||||
`expected one of the declared variants, got ${rendered}`,
|
||||
)
|
||||
assert.equal(f.vm.extraFieldDisplayPlaceholder({ key: 'k', label: 'k', type: 'string' }), '')
|
||||
} finally {
|
||||
f.close()
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import type { ModelProviderOption } from '@/api/initialization'
|
|||
import {
|
||||
credentialLabelForModelType,
|
||||
extraFieldLabel,
|
||||
extraFieldOptionLabel,
|
||||
extraFieldPlaceholder,
|
||||
extraFieldsForModelType,
|
||||
loadProvidersForType,
|
||||
mergeProviderIndex,
|
||||
|
|
@ -222,3 +224,37 @@ test('credentialLabelForModelType follows the vendor declaration, not a hardcode
|
|||
assert.equal(pickLocalized(rerank?.labels, 'zh-CN', rerank!.label), 'SecretId(TC3 签名)')
|
||||
assert.equal(pickLocalized(rerank?.labels, 'en-US', rerank!.label), 'SecretId')
|
||||
})
|
||||
|
||||
// Select options and placeholders used to render their raw English string.
|
||||
// That was invisible while every option was an identifier — LKEAP's region
|
||||
// codes — and became a gap as soon as one was prose.
|
||||
test('extraFieldOptionLabel resolves the locale, falling back to label then value', () => {
|
||||
const option = {
|
||||
label: '0..1 relevance (BGE class)',
|
||||
labels: { 'zh-CN': '0~1 相关度(BGE 一类)' },
|
||||
value: 'probability',
|
||||
}
|
||||
assert.equal(extraFieldOptionLabel(option, 'zh-CN'), '0~1 相关度(BGE 一类)')
|
||||
assert.equal(extraFieldOptionLabel(option, 'en'), '0..1 relevance (BGE class)')
|
||||
assert.equal(
|
||||
extraFieldOptionLabel({ label: 'ap-guangzhou', value: 'ap-guangzhou' }, 'zh-CN'),
|
||||
'ap-guangzhou',
|
||||
)
|
||||
assert.equal(extraFieldOptionLabel({ value: 'logit' }, 'zh-CN'), 'logit')
|
||||
})
|
||||
|
||||
test('extraFieldPlaceholder resolves the locale and tolerates no placeholder', () => {
|
||||
const field = {
|
||||
key: 'score_scale',
|
||||
label: 'Rerank score scale',
|
||||
type: 'select',
|
||||
placeholder: 'match the reranker actually deployed behind this endpoint',
|
||||
placeholders: { 'zh-CN': '按这个端点后面实际部署的重排模型选择' },
|
||||
}
|
||||
assert.equal(extraFieldPlaceholder(field, 'zh-CN'), '按这个端点后面实际部署的重排模型选择')
|
||||
assert.equal(
|
||||
extraFieldPlaceholder(field, 'en'),
|
||||
'match the reranker actually deployed behind this endpoint',
|
||||
)
|
||||
assert.equal(extraFieldPlaceholder({ key: 'k', label: 'k', type: 'string' }, 'zh-CN'), '')
|
||||
})
|
||||
|
|
|
|||
|
|
@ -109,6 +109,23 @@ export function extraFieldLabel(field: ModelProviderExtraField, locale?: string)
|
|||
return pickLocalized(field.labels, locale, field.label || field.key)
|
||||
}
|
||||
|
||||
/** Placeholder for the current locale, falling back to the default string. */
|
||||
export function extraFieldPlaceholder(field: ModelProviderExtraField, locale?: string): string {
|
||||
return pickLocalized(field.placeholders, locale, field.placeholder || '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Option label for the current locale. Select options used to render their
|
||||
* raw `label`, which was fine while every option was an identifier (a region
|
||||
* code) and wrong as soon as one was prose.
|
||||
*/
|
||||
export function extraFieldOptionLabel(
|
||||
option: { label?: string; labels?: Record<string, string>; value: string },
|
||||
locale?: string,
|
||||
): string {
|
||||
return pickLocalized(option.labels, locale, option.label || option.value)
|
||||
}
|
||||
|
||||
/** Stable display order: backend `order`, then label. */
|
||||
export function sortProviders(providers: ReadonlyArray<ModelProviderOption>): ModelProviderOption[] {
|
||||
return [...providers].sort((a, b) => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// BatchLimits are the per-request ceilings a vendor documents. A zero field
|
||||
// means the vendor states no limit, and nothing is enforced for it.
|
||||
type BatchLimits struct {
|
||||
// MaxItems caps how many documents one request may carry.
|
||||
MaxItems int
|
||||
// MaxItemRunes caps a single document. Vendors state these in characters
|
||||
// or tokens; runes are the closest thing we can count without a
|
||||
// tokenizer, and undercounting is the safe direction.
|
||||
MaxItemRunes int
|
||||
// MaxTotalRunes caps the whole request, query included.
|
||||
MaxTotalRunes int
|
||||
}
|
||||
|
||||
// Batch is one slice of the input, and where it started.
|
||||
type Batch struct {
|
||||
Start int
|
||||
Items []string
|
||||
}
|
||||
|
||||
// SplitBatches divides items into requests that satisfy limits. fixedRunes is
|
||||
// charged to every batch (the query, which each request repeats).
|
||||
//
|
||||
// A single item that cannot fit on its own is an error rather than a silent
|
||||
// truncation: the caller asked for this document to be scored, and scoring a
|
||||
// prefix of it returns a number for something the user never sent.
|
||||
func SplitBatches(items []string, fixedRunes int, limits BatchLimits) ([]Batch, error) {
|
||||
if limits.MaxTotalRunes > 0 && fixedRunes >= limits.MaxTotalRunes {
|
||||
return nil, fmt.Errorf(
|
||||
"query is %d characters; the request budget is %d and must also fit a document",
|
||||
fixedRunes, limits.MaxTotalRunes,
|
||||
)
|
||||
}
|
||||
|
||||
sizes := make([]int, len(items))
|
||||
for i, item := range items {
|
||||
sizes[i] = utf8.RuneCountInString(item)
|
||||
if limits.MaxItemRunes > 0 && sizes[i] > limits.MaxItemRunes {
|
||||
return nil, fmt.Errorf(
|
||||
"document at index %d is %d characters; the per-document limit is %d",
|
||||
i, sizes[i], limits.MaxItemRunes,
|
||||
)
|
||||
}
|
||||
if limits.MaxTotalRunes > 0 && fixedRunes+sizes[i] > limits.MaxTotalRunes {
|
||||
return nil, fmt.Errorf(
|
||||
"document at index %d is %d characters; it does not fit beside a %d-character query "+
|
||||
"in the %d-character request budget",
|
||||
i, sizes[i], fixedRunes, limits.MaxTotalRunes,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if limits.MaxItems <= 0 && limits.MaxTotalRunes <= 0 {
|
||||
return []Batch{{Start: 0, Items: items}}, nil
|
||||
}
|
||||
|
||||
batches := make([]Batch, 0, 1)
|
||||
start, total := 0, fixedRunes
|
||||
for i := range items {
|
||||
full := limits.MaxItems > 0 && i-start == limits.MaxItems
|
||||
over := limits.MaxTotalRunes > 0 && total+sizes[i] > limits.MaxTotalRunes
|
||||
if i > start && (full || over) {
|
||||
batches = append(batches, Batch{Start: start, Items: items[start:i]})
|
||||
start, total = i, fixedRunes
|
||||
}
|
||||
total += sizes[i]
|
||||
}
|
||||
batches = append(batches, Batch{Start: start, Items: items[start:]})
|
||||
return batches, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSplitBatchesNoLimitsKeepsOneRequest(t *testing.T) {
|
||||
got, err := SplitBatches([]string{"a", "b", "c"}, 0, BatchLimits{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, 0, got[0].Start)
|
||||
assert.Len(t, got[0].Items, 3)
|
||||
}
|
||||
|
||||
func TestSplitBatchesSplitsOnItemCount(t *testing.T) {
|
||||
items := []string{"a", "b", "c", "d", "e"}
|
||||
got, err := SplitBatches(items, 0, BatchLimits{MaxItems: 2})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 3)
|
||||
assert.Equal(t, []Batch{
|
||||
{Start: 0, Items: items[0:2]},
|
||||
{Start: 2, Items: items[2:4]},
|
||||
{Start: 4, Items: items[4:5]},
|
||||
}, got)
|
||||
}
|
||||
|
||||
// The query travels in every request, so it is charged to every batch rather
|
||||
// than once to the whole call.
|
||||
func TestSplitBatchesChargesTheQueryToEveryBatch(t *testing.T) {
|
||||
items := []string{strings.Repeat("x", 30), strings.Repeat("y", 30), strings.Repeat("z", 30)}
|
||||
got, err := SplitBatches(items, 40, BatchLimits{MaxTotalRunes: 100})
|
||||
require.NoError(t, err)
|
||||
// 40 + 30 + 30 = 100 fits; adding the third would reach 130.
|
||||
require.Len(t, got, 2)
|
||||
assert.Len(t, got[0].Items, 2)
|
||||
assert.Equal(t, 2, got[1].Start)
|
||||
}
|
||||
|
||||
func TestSplitBatchesRejectsAnItemThatCannotFit(t *testing.T) {
|
||||
_, err := SplitBatches([]string{"ok", strings.Repeat("x", 50)}, 0, BatchLimits{MaxItemRunes: 10})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "index 1")
|
||||
}
|
||||
|
||||
// A document that fits the per-document limit but not beside the query is
|
||||
// still unsendable; saying so beats looping forever or silently dropping it.
|
||||
func TestSplitBatchesRejectsAnItemThatCannotFitBesideTheQuery(t *testing.T) {
|
||||
_, err := SplitBatches([]string{strings.Repeat("x", 80)}, 40, BatchLimits{MaxTotalRunes: 100})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "does not fit beside")
|
||||
}
|
||||
|
||||
func TestSplitBatchesRejectsAQueryThatFillsTheBudget(t *testing.T) {
|
||||
_, err := SplitBatches([]string{"a"}, 100, BatchLimits{MaxTotalRunes: 100})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "must also fit a document")
|
||||
}
|
||||
|
||||
// Runes, not bytes: a CJK document is counted the way the vendor counts it.
|
||||
func TestSplitBatchesCountsRunesNotBytes(t *testing.T) {
|
||||
got, err := SplitBatches([]string{strings.Repeat("中", 10)}, 0, BatchLimits{MaxItemRunes: 10})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, 1)
|
||||
}
|
||||
|
||||
func TestSplitBatchesEmptyInput(t *testing.T) {
|
||||
got, err := SplitBatches(nil, 0, BatchLimits{MaxItems: 5})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
// Package cohererank implements the rerank shape Cohere defined and almost
|
||||
// every other vendor copied: POST {base}/rerank with {model, query,
|
||||
// documents} answering {results: [{index, relevance_score, document}]}.
|
||||
//
|
||||
// Jina, Zhipu, SiliconFlow, Qianfan, GPUStack, WeKnora Cloud and any
|
||||
// OpenAI-compatible gateway that serves rerank all speak it. Their
|
||||
// differences — which optional fields are accepted, where the endpoint sits,
|
||||
// what the documented ceilings are — arrive as catalog.RerankSettings; this
|
||||
// package contains no vendor names.
|
||||
package cohererank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
)
|
||||
|
||||
// Config is everything the client needs, already resolved by the catalog.
|
||||
type Config struct {
|
||||
Endpoint api.Endpoint
|
||||
Settings catalog.RerankSettings
|
||||
}
|
||||
|
||||
// Client talks the Cohere rerank shape to one endpoint.
|
||||
type Client struct {
|
||||
cfg Config
|
||||
}
|
||||
|
||||
// New builds a client.
|
||||
func New(cfg Config) *Client { return &Client{cfg: cfg} }
|
||||
|
||||
const defaultPath = "/rerank"
|
||||
|
||||
// url resolves the endpoint. A vendor whose default base URL already names
|
||||
// the full rerank endpoint — and an operator who pasted one — must not have
|
||||
// the path appended twice.
|
||||
func (c *Client) url() string {
|
||||
if c.cfg.Endpoint.URL != "" {
|
||||
return c.cfg.Endpoint.Resolve("")
|
||||
}
|
||||
path := c.cfg.Settings.Path
|
||||
if path == "" {
|
||||
path = defaultPath
|
||||
}
|
||||
if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
|
||||
path = ""
|
||||
}
|
||||
return c.cfg.Endpoint.Resolve(path)
|
||||
}
|
||||
|
||||
type request struct {
|
||||
Model string `json:"model"`
|
||||
Query string `json:"query"`
|
||||
Documents []string `json:"documents"`
|
||||
TopN int `json:"top_n,omitempty"`
|
||||
// TruncatePromptTokens is a vLLM extension, not part of any vendor's
|
||||
// documented schema. It is sent only when the operator opted in, because
|
||||
// a gateway that does not implement it rejects the unknown field.
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"`
|
||||
ReturnDocuments *bool `json:"return_documents,omitempty"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Results []result `json:"results"`
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore *float64 `json:"relevance_score"`
|
||||
Score *float64 `json:"score"`
|
||||
Document document `json:"document"`
|
||||
}
|
||||
|
||||
// document is the echoed text, which vendors spell either as a bare string
|
||||
// or as an object with a text field.
|
||||
type document struct {
|
||||
Text string
|
||||
}
|
||||
|
||||
func (d *document) UnmarshalJSON(data []byte) error {
|
||||
var text string
|
||||
if err := json.Unmarshal(data, &text); err == nil {
|
||||
d.Text = text
|
||||
return nil
|
||||
}
|
||||
var obj struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &obj); err != nil {
|
||||
return fmt.Errorf("decode document: %w", err)
|
||||
}
|
||||
d.Text = obj.Text
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildRequestBody is the golden-test entry point: it returns the exact JSON
|
||||
// object that would be sent.
|
||||
func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) {
|
||||
body := request{
|
||||
Model: c.cfg.Endpoint.Model,
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
}
|
||||
if c.cfg.Settings.AcceptsTruncatePromptTokens {
|
||||
body.TruncatePromptTokens = c.cfg.Settings.TruncatePromptTokens
|
||||
}
|
||||
if c.cfg.Settings.SendTopN {
|
||||
body.TopN = len(documents)
|
||||
}
|
||||
if c.cfg.Settings.SendReturnDocs {
|
||||
enabled := true
|
||||
body.ReturnDocuments = &enabled
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
for k, v := range c.cfg.Settings.ExtraBody {
|
||||
if _, exists := out[k]; !exists {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Rerank scores documents against the query, in the order they were given.
|
||||
func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) {
|
||||
body, err := c.BuildRequestBody(query, documents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded response
|
||||
if err := c.cfg.Endpoint.PostJSON(ctx, c.url(), body, &decoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]api.RerankResult, 0, len(decoded.Results))
|
||||
for _, item := range decoded.Results {
|
||||
if item.Index < 0 || item.Index >= len(documents) {
|
||||
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents))
|
||||
}
|
||||
out = append(out, api.RerankResult{
|
||||
Index: item.Index,
|
||||
Score: firstNonNil(item.RelevanceScore, item.Score),
|
||||
Text: item.Document.Text,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// firstNonNil picks relevance_score, falling back to score: the field is
|
||||
// spelled both ways across vendors that otherwise share this shape.
|
||||
func firstNonNil(values ...*float64) float64 {
|
||||
for _, v := range values {
|
||||
if v != nil {
|
||||
return *v
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
package cohererank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// zhipuResponse is the documented response of Zhipu's rerank endpoint, whose
|
||||
// result objects carry document / index / relevance_score and whose envelope
|
||||
// adds id, request_id, created and usage:
|
||||
// https://docs.bigmodel.cn/api-reference/模型-api/文本重排序
|
||||
const zhipuResponse = `{
|
||||
"id": "8305283656248111323",
|
||||
"request_id": "8305283656248111323",
|
||||
"created": 1769040000,
|
||||
"results": [
|
||||
{"index": 1, "relevance_score": 0.9819, "document": "上海气候"},
|
||||
{"index": 0, "relevance_score": 0.0021, "document": "北京美食"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 24, "total_tokens": 24}
|
||||
}`
|
||||
|
||||
// jinaResponse spells the echoed document as an object instead of a string,
|
||||
// which is the other shape vendors use for the same field:
|
||||
// https://jina.ai/reranker/
|
||||
const jinaResponse = `{
|
||||
"model": "jina-reranker-v3.5",
|
||||
"results": [
|
||||
{"index": 0, "relevance_score": 0.95, "document": {"text": "上海气候"}},
|
||||
{"index": 1, "relevance_score": 0.11, "document": {"text": "北京美食"}}
|
||||
],
|
||||
"usage": {"total_tokens": 32}
|
||||
}`
|
||||
|
||||
func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client {
|
||||
t.Helper()
|
||||
return New(Config{
|
||||
Endpoint: api.Endpoint{BaseURL: url, Model: "rerank", Auth: api.BearerAuth("k")},
|
||||
Settings: settings,
|
||||
})
|
||||
}
|
||||
|
||||
func serve(t *testing.T, payload string) (*httptest.Server, *string, *map[string]any) {
|
||||
t.Helper()
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
path := new(string)
|
||||
body := new(map[string]any)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
*path = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(payload))
|
||||
}))
|
||||
return server, path, body
|
||||
}
|
||||
|
||||
func TestRequestBodyCarriesOnlyWhatTheVendorDeclares(t *testing.T) {
|
||||
c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{})
|
||||
body, err := c.BuildRequestBody("上海天气", []string{"上海气候", "北京美食"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": "rerank",
|
||||
"query": "上海天气",
|
||||
"documents": []any{"上海气候", "北京美食"},
|
||||
}, body, "optional fields must be absent unless the vendor asked for them")
|
||||
}
|
||||
|
||||
func TestRequestBodyAddsTheOptionalFieldsAVendorDeclares(t *testing.T) {
|
||||
c := newClient(t, "https://example.invalid/v1",
|
||||
catalog.RerankSettings{SendTopN: true, SendReturnDocs: true})
|
||||
body, err := c.BuildRequestBody("q", []string{"a", "b", "c"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, float64(3), body["top_n"], "top_n asks for every document back")
|
||||
assert.Equal(t, true, body["return_documents"])
|
||||
}
|
||||
|
||||
func TestDecodesAStringDocument(t *testing.T) {
|
||||
server, _, _ := serve(t, zhipuResponse)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
got, err := c.Rerank(context.Background(), "上海天气", []string{"北京美食", "上海气候"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []api.RerankResult{
|
||||
{Index: 1, Score: 0.9819, Text: "上海气候"},
|
||||
{Index: 0, Score: 0.0021, Text: "北京美食"},
|
||||
}, got)
|
||||
}
|
||||
|
||||
func TestDecodesAnObjectDocument(t *testing.T) {
|
||||
server, _, _ := serve(t, jinaResponse)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
got, err := c.Rerank(context.Background(), "上海天气", []string{"上海气候", "北京美食"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []api.RerankResult{
|
||||
{Index: 0, Score: 0.95, Text: "上海气候"},
|
||||
{Index: 1, Score: 0.11, Text: "北京美食"},
|
||||
}, got)
|
||||
}
|
||||
|
||||
// Some gateways serving this shape spell the score `score`. The pre-catalog
|
||||
// client accepted both and so does this one.
|
||||
func TestDecodesTheScoreAlias(t *testing.T) {
|
||||
server, _, _ := serve(t, `{"results":[{"index":0,"score":0.42}]}`)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
got, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
assert.InDelta(t, 0.42, got[0].Score, 1e-9)
|
||||
}
|
||||
|
||||
func TestAppendsTheRerankPathToABareBaseURL(t *testing.T) {
|
||||
server, path, _ := serve(t, `{"results":[]}`)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL+"/v1", catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/v1/rerank", *path)
|
||||
}
|
||||
|
||||
// Zhipu's default base URL already ends in /rerank, and operators paste full
|
||||
// endpoints too. Appending the path again would 404.
|
||||
func TestDoesNotDoubleTheRerankPath(t *testing.T) {
|
||||
server, path, _ := serve(t, `{"results":[]}`)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL+"/api/paas/v4/rerank", catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/api/paas/v4/rerank", *path)
|
||||
}
|
||||
|
||||
// WeKnora Cloud serves the same shape on its own path.
|
||||
func TestHonoursAVendorDeclaredPath(t *testing.T) {
|
||||
server, path, _ := serve(t, `{"results":[]}`)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{Path: "/api/v1/rerank"})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/api/v1/rerank", *path)
|
||||
}
|
||||
|
||||
func TestRejectsAnOutOfRangeIndex(t *testing.T) {
|
||||
server, _, _ := serve(t, `{"results":[{"index":9,"relevance_score":0.5}]}`)
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "out of range")
|
||||
}
|
||||
|
||||
func TestSurfacesTheVendorErrorBody(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusUnprocessableEntity)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"documents exceeds the limit"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "documents exceeds the limit")
|
||||
}
|
||||
|
||||
// truncate_prompt_tokens is a vLLM extension that appears in no managed
|
||||
// vendor's documented schema. It is sent only where the vendor declared it
|
||||
// and the operator opted in, and never by default — sending it unasked is how
|
||||
// an undocumented field ends up on every request to every gateway.
|
||||
func TestTruncatePromptTokensIsAbsentByDefault(t *testing.T) {
|
||||
c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{})
|
||||
body, err := c.BuildRequestBody("q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, body, "truncate_prompt_tokens")
|
||||
}
|
||||
|
||||
func TestTruncatePromptTokensIsSentWhenTheRowOptedIn(t *testing.T) {
|
||||
c := newClient(t, "https://example.invalid/v1", catalog.RerankSettings{
|
||||
AcceptsTruncatePromptTokens: true, TruncatePromptTokens: 512,
|
||||
})
|
||||
body, err := c.BuildRequestBody("q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, float64(512), body["truncate_prompt_tokens"])
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
// Package dashscoperank implements Alibaba Model Studio's native text-rerank
|
||||
// shape: the same fields as the Cohere dialect, wrapped in input/parameters
|
||||
// and answered under output.
|
||||
//
|
||||
// https://help.aliyun.com/zh/model-studio/text-rerank-api
|
||||
package dashscoperank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
)
|
||||
|
||||
// Config is everything the client needs, already resolved by the catalog.
|
||||
type Config struct {
|
||||
Endpoint api.Endpoint
|
||||
Settings catalog.RerankSettings
|
||||
}
|
||||
|
||||
// Client talks DashScope's native rerank to one endpoint.
|
||||
type Client struct {
|
||||
cfg Config
|
||||
}
|
||||
|
||||
// New builds a client.
|
||||
func New(cfg Config) *Client { return &Client{cfg: cfg} }
|
||||
|
||||
type request struct {
|
||||
Model string `json:"model"`
|
||||
Input input `json:"input"`
|
||||
Parameters parameters `json:"parameters"`
|
||||
}
|
||||
|
||||
type input struct {
|
||||
Query string `json:"query"`
|
||||
Documents []string `json:"documents"`
|
||||
}
|
||||
|
||||
type parameters struct {
|
||||
ReturnDocuments bool `json:"return_documents"`
|
||||
TopN int `json:"top_n"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Output struct {
|
||||
Results []struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore float64 `json:"relevance_score"`
|
||||
Document struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"document"`
|
||||
} `json:"results"`
|
||||
} `json:"output"`
|
||||
// DashScope reports failures in the body with a 200 on some paths.
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// BuildRequestBody is the golden-test entry point.
|
||||
func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) {
|
||||
body := request{
|
||||
Model: c.cfg.Endpoint.Model,
|
||||
Input: input{Query: query, Documents: documents},
|
||||
Parameters: parameters{
|
||||
ReturnDocuments: c.cfg.Settings.SendReturnDocs,
|
||||
// top_n is not optional in this shape: leaving it at zero would
|
||||
// ask for no documents at all, so the full count is always sent.
|
||||
TopN: len(documents),
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
for k, v := range c.cfg.Settings.ExtraBody {
|
||||
if _, exists := out[k]; !exists {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Rerank scores documents against the query.
|
||||
func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) {
|
||||
body, err := c.BuildRequestBody(query, documents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded response
|
||||
url := c.cfg.Endpoint.Resolve(c.cfg.Settings.Path)
|
||||
if err := c.cfg.Endpoint.PostJSON(ctx, url, body, &decoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if decoded.Code != "" {
|
||||
return nil, fmt.Errorf("DashScope rerank error %s: %s", decoded.Code, decoded.Message)
|
||||
}
|
||||
out := make([]api.RerankResult, 0, len(decoded.Output.Results))
|
||||
for _, item := range decoded.Output.Results {
|
||||
if item.Index < 0 || item.Index >= len(documents) {
|
||||
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents))
|
||||
}
|
||||
out = append(out, api.RerankResult{
|
||||
Index: item.Index,
|
||||
Score: item.RelevanceScore,
|
||||
Text: item.Document.Text,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
package dashscoperank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// documentedResponse is the response shape of DashScope's text-rerank API,
|
||||
// copied from the reference so the decoder is pinned against the vendor's own
|
||||
// document: https://help.aliyun.com/zh/model-studio/text-rerank-api
|
||||
const documentedResponse = `{
|
||||
"output": {
|
||||
"results": [
|
||||
{"document": {"text": "上海气候"}, "index": 1, "relevance_score": 0.7314},
|
||||
{"document": {"text": "北京美食"}, "index": 0, "relevance_score": 0.0002}
|
||||
]
|
||||
},
|
||||
"usage": {"total_tokens": 24},
|
||||
"request_id": "4c5b1c4d-0e2f-9b1a-8a1e-1b2c3d4e5f60"
|
||||
}`
|
||||
|
||||
const endpointURL = "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
|
||||
|
||||
func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client {
|
||||
t.Helper()
|
||||
return New(Config{
|
||||
Endpoint: api.Endpoint{BaseURL: url, Model: "gte-rerank-v2", Auth: api.BearerAuth("k")},
|
||||
Settings: settings,
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestBodyMatchesTheDocumentedSchema(t *testing.T) {
|
||||
c := newClient(t, endpointURL, catalog.RerankSettings{SendReturnDocs: true})
|
||||
body, err := c.BuildRequestBody("上海天气", []string{"北京美食", "上海气候"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": "gte-rerank-v2",
|
||||
"input": map[string]any{
|
||||
"query": "上海天气",
|
||||
"documents": []any{"北京美食", "上海气候"},
|
||||
},
|
||||
"parameters": map[string]any{
|
||||
"return_documents": true,
|
||||
"top_n": float64(2),
|
||||
},
|
||||
}, body)
|
||||
}
|
||||
|
||||
// top_n is mandatory in this shape: zero would ask for nothing back, so the
|
||||
// full document count is always sent regardless of vendor settings.
|
||||
func TestTopNAlwaysCarriesTheDocumentCount(t *testing.T) {
|
||||
c := newClient(t, endpointURL, catalog.RerankSettings{})
|
||||
body, err := c.BuildRequestBody("q", []string{"a", "b", "c", "d"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, float64(4), body["parameters"].(map[string]any)["top_n"])
|
||||
}
|
||||
|
||||
func TestDecodesTheDocumentedResponse(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(documentedResponse))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
got, err := c.Rerank(context.Background(), "上海天气", []string{"北京美食", "上海气候"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []api.RerankResult{
|
||||
{Index: 1, Score: 0.7314, Text: "上海气候"},
|
||||
{Index: 0, Score: 0.0002, Text: "北京美食"},
|
||||
}, got)
|
||||
}
|
||||
|
||||
// DashScope answers some failures with a 200 and an error code in the body,
|
||||
// which would otherwise decode as an empty result set.
|
||||
func TestSurfacesAnInBodyError(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"code":"InvalidParameter","message":"documents is too long","request_id":"r"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "documents is too long")
|
||||
}
|
||||
|
||||
func TestPostsToTheBaseURLItself(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"output":{"results":[]}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL+"/api/v1/services/rerank/text-rerank/text-rerank", catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/api/v1/services/rerank/text-rerank/text-rerank", gotPath,
|
||||
"the vendor's base URL already names the full endpoint")
|
||||
}
|
||||
|
|
@ -135,6 +135,40 @@ func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
|
|||
return resp, nil
|
||||
}
|
||||
|
||||
// PostJSON sends one authenticated JSON request and decodes the reply into
|
||||
// out. It is the whole round-trip for the non-streaming protocols (rerank,
|
||||
// embeddings), which have no SSE to assemble.
|
||||
//
|
||||
// It never returns a nil error with nothing decoded: every failure path —
|
||||
// marshalling, SSRF, transport, non-2xx, decoding — returns an error, so a
|
||||
// caller cannot mistake an empty result for a successful empty answer.
|
||||
func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error {
|
||||
req, _, err := e.NewRequest(ctx, url, body, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// No LogRequest here: the chat protocols log their own bodies because a
|
||||
// prompt is what an operator debugs, while a rerank body is a query plus
|
||||
// every candidate chunk. The rerank layer logs a truncated line at Debug
|
||||
// instead, so this would have duplicated it at Info.
|
||||
resp, err := e.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, out); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HTTPError is a non-2xx vendor reply.
|
||||
type HTTPError struct {
|
||||
StatusCode int
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
// Package nimrerank implements NVIDIA NIM's retrieval reranking shape: a
|
||||
// query object, a passages array, and rankings carrying a logit.
|
||||
//
|
||||
// Two things separate it from the Cohere dialect and both matter to callers.
|
||||
// The score is an unbounded log-odds value that is routinely negative, not a
|
||||
// 0..1 relevance — catalog.RerankSettings.ScoreScale says so. And `truncate`
|
||||
// defaults to "NONE", which means an over-long passage fails the request
|
||||
// instead of being cut, so the vendor sets it explicitly.
|
||||
//
|
||||
// https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer
|
||||
package nimrerank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
)
|
||||
|
||||
// Config is everything the client needs, already resolved by the catalog.
|
||||
type Config struct {
|
||||
Endpoint api.Endpoint
|
||||
Settings catalog.RerankSettings
|
||||
}
|
||||
|
||||
// Client talks NIM reranking to one endpoint.
|
||||
type Client struct {
|
||||
cfg Config
|
||||
}
|
||||
|
||||
// New builds a client.
|
||||
func New(cfg Config) *Client { return &Client{cfg: cfg} }
|
||||
|
||||
type text struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type request struct {
|
||||
Model string `json:"model"`
|
||||
Query text `json:"query"`
|
||||
Passages []text `json:"passages"`
|
||||
Truncate string `json:"truncate,omitempty"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Rankings []struct {
|
||||
Index int `json:"index"`
|
||||
Logit float64 `json:"logit"`
|
||||
} `json:"rankings"`
|
||||
}
|
||||
|
||||
// BuildRequestBody is the golden-test entry point.
|
||||
func (c *Client) BuildRequestBody(query string, documents []string) (map[string]any, error) {
|
||||
passages := make([]text, len(documents))
|
||||
for i := range documents {
|
||||
passages[i] = text{Text: documents[i]}
|
||||
}
|
||||
body := request{
|
||||
Model: c.cfg.Endpoint.Model,
|
||||
Query: text{Text: query},
|
||||
Passages: passages,
|
||||
Truncate: c.cfg.Settings.Truncate,
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
for k, v := range c.cfg.Settings.ExtraBody {
|
||||
if _, exists := out[k]; !exists {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Rerank scores documents against the query. The returned scores are logits;
|
||||
// see the package comment.
|
||||
func (c *Client) Rerank(ctx context.Context, query string, documents []string) ([]api.RerankResult, error) {
|
||||
body, err := c.BuildRequestBody(query, documents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded response
|
||||
url := c.cfg.Endpoint.Resolve(c.cfg.Settings.Path)
|
||||
if err := c.cfg.Endpoint.PostJSON(ctx, url, body, &decoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]api.RerankResult, 0, len(decoded.Rankings))
|
||||
for _, item := range decoded.Rankings {
|
||||
if item.Index < 0 || item.Index >= len(documents) {
|
||||
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, len(documents))
|
||||
}
|
||||
out = append(out, api.RerankResult{
|
||||
Index: item.Index,
|
||||
Score: item.Logit,
|
||||
Text: documents[item.Index],
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
package nimrerank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// documentedResponse is the response example from NVIDIA's reranking
|
||||
// reference, copied verbatim so the decoder is pinned against the vendor's
|
||||
// own document rather than against our structs:
|
||||
// https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer
|
||||
//
|
||||
// Note the negative logit. This protocol does not return a 0..1 relevance
|
||||
// score, which is why catalog.RerankSettings.ScoreScale exists.
|
||||
const documentedResponse = `{
|
||||
"rankings": [
|
||||
{"index": 2, "logit": 0.226318359375},
|
||||
{"index": 1, "logit": -1.171875},
|
||||
{"index": 0, "logit": -6.3125}
|
||||
]
|
||||
}`
|
||||
|
||||
func newClient(t *testing.T, url string, settings catalog.RerankSettings) *Client {
|
||||
t.Helper()
|
||||
return New(Config{
|
||||
Endpoint: api.Endpoint{BaseURL: url, Model: "nvidia/nv-rerankqa-mistral-4b-v3", Auth: api.BearerAuth("k")},
|
||||
Settings: settings,
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestBodyMatchesTheDocumentedSchema(t *testing.T) {
|
||||
c := newClient(t, "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking",
|
||||
catalog.RerankSettings{Truncate: "END", ScoreScale: api.ScoreLogit})
|
||||
|
||||
body, err := c.BuildRequestBody("q", []string{"d0", "d1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": "nvidia/nv-rerankqa-mistral-4b-v3",
|
||||
"query": map[string]any{"text": "q"},
|
||||
"passages": []any{map[string]any{"text": "d0"}, map[string]any{"text": "d1"}},
|
||||
"truncate": "END",
|
||||
}, body)
|
||||
}
|
||||
|
||||
// truncate defaults to NONE upstream, which fails the request on an over-long
|
||||
// passage instead of cutting it. A vendor that does not set it must not have
|
||||
// the key invented for it either.
|
||||
func TestTruncateIsOmittedWhenUnset(t *testing.T) {
|
||||
c := newClient(t, "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking", catalog.RerankSettings{})
|
||||
body, err := c.BuildRequestBody("q", []string{"d0"})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, body, "truncate")
|
||||
}
|
||||
|
||||
func TestDecodesTheDocumentedResponse(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(documentedResponse))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{ScoreScale: api.ScoreLogit})
|
||||
got, err := c.Rerank(context.Background(), "q", []string{"d0", "d1", "d2"})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []api.RerankResult{
|
||||
{Index: 2, Score: 0.226318359375, Text: "d2"},
|
||||
{Index: 1, Score: -1.171875, Text: "d1"},
|
||||
{Index: 0, Score: -6.3125, Text: "d0"},
|
||||
}, got)
|
||||
}
|
||||
|
||||
// An index the request never sent is a protocol violation, not something to
|
||||
// map onto whatever document happens to sit at that offset.
|
||||
func TestRejectsAnOutOfRangeIndex(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"rankings":[{"index":7,"logit":0.5}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d0"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "out of range")
|
||||
}
|
||||
|
||||
func TestSurfacesTheVendorErrorBody(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"detail":"input too long"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL, catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d0"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "input too long")
|
||||
}
|
||||
|
||||
func TestRequestReachesTheConfiguredURL(t *testing.T) {
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"rankings":[]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
c := newClient(t, server.URL+"/v1/retrieval/nvidia/reranking", catalog.RerankSettings{})
|
||||
_, err := c.Rerank(context.Background(), "q", []string{"d0"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/v1/retrieval/nvidia/reranking", gotPath, "the base URL already names the endpoint")
|
||||
assert.Equal(t, "q", gotBody["query"].(map[string]any)["text"])
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
package api
|
||||
|
||||
import "context"
|
||||
|
||||
// RerankAPI names a rerank wire protocol. It is deliberately a separate type
|
||||
// from API, the chat protocol selector: the two vocabularies must not mix, or
|
||||
// a chat row whose extra_config.api names a rerank dialect would pass
|
||||
// validation and fail at call time.
|
||||
type RerankAPI string
|
||||
|
||||
// The rerank protocols WeKnora speaks.
|
||||
const (
|
||||
// RerankCohere is the de-facto standard shape — POST {base}/rerank with
|
||||
// {model, query, documents} and a results array of {index,
|
||||
// relevance_score, document}. Cohere defined it; Jina, Zhipu,
|
||||
// SiliconFlow, Qianfan, GPUStack and every OpenAI-compatible gateway
|
||||
// that serves rerank at all copy it.
|
||||
RerankCohere RerankAPI = "cohere-rerank"
|
||||
// RerankDashScope is Alibaba Model Studio's native shape, which wraps
|
||||
// the same fields in input/parameters and answers under output.
|
||||
RerankDashScope RerankAPI = "dashscope-rerank"
|
||||
// RerankNIM is NVIDIA NIM's retrieval shape: a query object, a passages
|
||||
// array, and rankings carrying an unbounded logit instead of a
|
||||
// probability.
|
||||
RerankNIM RerankAPI = "nim-rerank"
|
||||
// RerankTencentLKEAP is Tencent Cloud's RunRerank action, reached through
|
||||
// the official SDK because it is TC3-signed rather than key-authenticated.
|
||||
RerankTencentLKEAP RerankAPI = "tencent-lkeap"
|
||||
// RerankVolcengineKnowledge is Volcengine's managed Knowledge Service
|
||||
// rerank, reached through the vikingdb SDK for the same reason.
|
||||
RerankVolcengineKnowledge RerankAPI = "volcengine-knowledge"
|
||||
)
|
||||
|
||||
// Known reports whether the value names a protocol this build implements.
|
||||
func (a RerankAPI) Known() bool {
|
||||
switch a {
|
||||
case RerankCohere, RerankDashScope, RerankNIM,
|
||||
RerankTencentLKEAP, RerankVolcengineKnowledge:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HTTPServed reports whether a protocol package implements the protocol. The
|
||||
// SDK-backed ones are built by the rerank package itself, which is where the
|
||||
// vendor SDKs are imported.
|
||||
func (a RerankAPI) HTTPServed() bool {
|
||||
switch a {
|
||||
case RerankCohere, RerankDashScope, RerankNIM:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ScoreScale describes what a rerank score means, which differs per protocol
|
||||
// and cannot be inferred from the number itself.
|
||||
type ScoreScale string
|
||||
|
||||
const (
|
||||
// ScoreProbability is a 0..1 relevance score. Every protocol but NIM
|
||||
// returns one.
|
||||
ScoreProbability ScoreScale = "probability"
|
||||
// ScoreLogit is an unbounded log-odds value that may be negative. NIM
|
||||
// returns one, so a threshold tuned for probabilities rejects almost
|
||||
// everything on that vendor.
|
||||
ScoreLogit ScoreScale = "logit"
|
||||
)
|
||||
|
||||
// RerankResult is one scored document, in the vendor's own score scale.
|
||||
// Index refers to the position in the documents slice that was sent.
|
||||
type RerankResult struct {
|
||||
Index int
|
||||
Score float64
|
||||
// Text is the document echoed back, empty when the vendor was not asked
|
||||
// to return documents. Callers index into their own slice instead of
|
||||
// relying on it.
|
||||
Text string
|
||||
}
|
||||
|
||||
// Reranker is what every rerank protocol package implements.
|
||||
type Reranker interface {
|
||||
Rerank(ctx context.Context, query string, documents []string) ([]RerankResult, error)
|
||||
}
|
||||
|
|
@ -341,6 +341,9 @@ type VendorCompat struct {
|
|||
OpenAIResponses OpenAIResponsesCompat `json:"openai_responses,omitempty"`
|
||||
AnthropicMessages AnthropicMessagesCompat `json:"anthropic_messages,omitempty"`
|
||||
GoogleGenerativeAI GoogleGenerativeAICompat `json:"google_generative_ai,omitempty"`
|
||||
// Rerank is the rerank protocol overlay. It has no per-protocol variants:
|
||||
// a vendor serves exactly one rerank dialect.
|
||||
Rerank RerankCompat `json:"rerank,omitempty"`
|
||||
}
|
||||
|
||||
// apply writes the set fields of an overlay struct onto a settings struct
|
||||
|
|
|
|||
|
|
@ -194,7 +194,14 @@ func TestLegacyRowsValidateAndResolve(t *testing.T) {
|
|||
if resolved.Vendor == nil {
|
||||
t.Fatal("resolved with a nil vendor")
|
||||
}
|
||||
if !resolved.API.Known() {
|
||||
// Rerank rows resolve to a rerank protocol; everything else
|
||||
// resolves to a chat one. The two vocabularies are separate
|
||||
// types so a row can never land on the wrong one.
|
||||
if row.typ == types.ModelTypeRerank {
|
||||
if !resolved.RerankAPI.Known() {
|
||||
t.Fatalf("resolved to an unknown rerank protocol %q", resolved.RerankAPI)
|
||||
}
|
||||
} else if !resolved.API.Known() {
|
||||
t.Fatalf("resolved to an unknown protocol %q", resolved.API)
|
||||
}
|
||||
if resolved.RemoteModel == "" && row.model != "" {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ func Register(v *Vendor) {
|
|||
if v.Auth == "" {
|
||||
v.Auth = AuthBearer
|
||||
}
|
||||
if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) {
|
||||
v.RerankAPI = api.RerankCohere
|
||||
}
|
||||
for i := range v.Models {
|
||||
if v.Models[i].Type == "" {
|
||||
v.Models[i].Type = types.ModelTypeKnowledgeQA
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
package catalog
|
||||
|
||||
import "github.com/Tencent/WeKnora/internal/models/api"
|
||||
|
||||
// RerankCompat is the overlay form (every field optional) of the rerank
|
||||
// settings. JSON keys are the documented names used in models.json and
|
||||
// config/models.json.
|
||||
type RerankCompat struct {
|
||||
Path *string `json:"path,omitempty"`
|
||||
SendTopN *bool `json:"send_top_n,omitempty"`
|
||||
SendReturnDocs *bool `json:"send_return_documents,omitempty"`
|
||||
ScoreScale *api.ScoreScale `json:"score_scale,omitempty"`
|
||||
Truncate *string `json:"truncate,omitempty"`
|
||||
MaxDocuments *int `json:"max_documents,omitempty"`
|
||||
MaxQueryChars *int `json:"max_query_chars,omitempty"`
|
||||
MaxDocumentChars *int `json:"max_document_chars,omitempty"`
|
||||
MaxRequestChars *int `json:"max_request_chars,omitempty"`
|
||||
MaxConcurrency *int `json:"max_concurrency,omitempty"`
|
||||
RequestTimeout *int `json:"request_timeout_seconds,omitempty"`
|
||||
// AcceptsTruncatePromptTokens marks a vLLM-class runtime.
|
||||
AcceptsTruncatePromptTokens *bool `json:"accepts_truncate_prompt_tokens,omitempty"`
|
||||
// UnsupportedReason marks a model this build cannot call.
|
||||
UnsupportedReason *string `json:"unsupported_reason,omitempty"`
|
||||
ExtraBody map[string]any `json:"extra_body,omitempty"`
|
||||
}
|
||||
|
||||
// RerankSettings is the resolved (fully defaulted) form.
|
||||
type RerankSettings struct {
|
||||
// Path is appended to the base URL by protocols that use one. Vendors
|
||||
// whose default base URL already names the full endpoint leave it empty.
|
||||
Path string
|
||||
// SendTopN asks for every document back by naming their count. Omitting
|
||||
// it means "all", which is what every protocol here does by default.
|
||||
SendTopN bool
|
||||
// SendReturnDocs asks the vendor to echo the document text. Callers index
|
||||
// into their own slice, so this is off unless a vendor needs it.
|
||||
SendReturnDocs bool
|
||||
// ScoreScale says what the returned numbers mean. See api.ScoreScale:
|
||||
// getting this wrong makes a relevance threshold meaningless.
|
||||
ScoreScale api.ScoreScale
|
||||
// Truncate is the vendor's server-side truncation setting for
|
||||
// over-long input ("END" on NIM). Empty sends nothing.
|
||||
Truncate string
|
||||
// Limits carry the documented per-request ceilings. MaxQueryChars is
|
||||
// checked on its own rather than through BatchLimits: the query is not an
|
||||
// item to be split, so an over-long one cannot be made to fit by batching.
|
||||
MaxDocuments int
|
||||
MaxQueryChars int
|
||||
MaxDocumentChars int
|
||||
MaxRequestChars int
|
||||
// MaxConcurrency bounds in-flight batches when the input is split. 0
|
||||
// means the caller's default.
|
||||
MaxConcurrency int
|
||||
// RequestTimeout caps one request, in seconds. 0 leaves the HTTP client
|
||||
// without its own deadline and lets the caller's context govern, which is
|
||||
// what every vendor but WeKnora Cloud has always done here.
|
||||
RequestTimeout int
|
||||
// AcceptsTruncatePromptTokens reports that this vendor is a vLLM-class
|
||||
// runtime, which is the only kind that implements the
|
||||
// `truncate_prompt_tokens` extension. It is not part of the Cohere shape
|
||||
// and appears in no managed vendor's schema, so it must never be sent to
|
||||
// one: that is how an undocumented field ends up on every request.
|
||||
AcceptsTruncatePromptTokens bool
|
||||
// UnsupportedReason is set on a catalog entry naming a rerank dialect no
|
||||
// protocol package implements. Resolve refuses it rather than letting a
|
||||
// request go out shaped for a different protocol, which surfaces as a
|
||||
// decode error far from its cause.
|
||||
UnsupportedReason string
|
||||
// TruncatePromptTokens is the budget the operator opted into on this row,
|
||||
// honoured only where AcceptsTruncatePromptTokens is set. It is a row
|
||||
// setting rather than a vendor fact because one runtime serves models
|
||||
// with different context windows. Zero sends nothing.
|
||||
TruncatePromptTokens int
|
||||
ExtraBody map[string]any
|
||||
}
|
||||
|
||||
// BatchLimits renders the documented ceilings for api.SplitBatches.
|
||||
func (s RerankSettings) BatchLimits() api.BatchLimits {
|
||||
return api.BatchLimits{
|
||||
MaxItems: s.MaxDocuments,
|
||||
MaxItemRunes: s.MaxDocumentChars,
|
||||
MaxTotalRunes: s.MaxRequestChars,
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultRerank is the protocol baseline: ask for every document, expect a
|
||||
// 0..1 relevance score, enforce no ceiling the vendor did not state.
|
||||
func DefaultRerank() RerankSettings {
|
||||
return RerankSettings{
|
||||
ScoreScale: api.ScoreProbability,
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ package catalog
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
|
|
@ -21,6 +22,16 @@ const (
|
|||
// ExtraThinkingControl is the legacy thinking encoding selector written
|
||||
// by older UIs: none | enable_thinking | thinking_type | chat_template_kwargs.
|
||||
ExtraThinkingControl = "thinking_control"
|
||||
// ExtraTruncatePromptTokens is the vLLM-only server-side truncation
|
||||
// budget for rerank, opt-in per row. It is never sent unless the operator
|
||||
// set it: vendors that do not implement it reject the unknown field.
|
||||
ExtraTruncatePromptTokens = "truncate_prompt_tokens"
|
||||
// ExtraScoreScale overrides the vendor rerank score scale for one row.
|
||||
// A self-hosted gateway serves whatever reranker was deployed behind it
|
||||
// and the two families disagree: BGE-class models answer a 0..1
|
||||
// probability, Qwen3-Reranker-class models an unbounded score. A vendor
|
||||
// can only state what its own documentation shows.
|
||||
ExtraScoreScale = "score_scale"
|
||||
)
|
||||
|
||||
// Ref identifies one configured model.
|
||||
|
|
@ -51,6 +62,12 @@ type Resolved struct {
|
|||
OpenAIResponses OpenAIResponsesSettings
|
||||
AnthropicMessages AnthropicMessagesSettings
|
||||
GoogleGenerativeAI GoogleGenerativeAISettings
|
||||
|
||||
// RerankAPI and Rerank are filled only when the reference asked for a
|
||||
// rerank model. Chat resolution is on every request's hot path, so the
|
||||
// rerank overlay is not merged for it.
|
||||
RerankAPI api.RerankAPI
|
||||
Rerank RerankSettings
|
||||
}
|
||||
|
||||
// Resolve merges the vendor, catalog entry, extra-config and per-row
|
||||
|
|
@ -86,6 +103,10 @@ func Resolve(ref Ref) (*Resolved, error) {
|
|||
applySpecOverride(&spec, ref.Override)
|
||||
}
|
||||
|
||||
if modelType == types.ModelTypeRerank {
|
||||
return resolveRerank(ref, vendor, spec, cataloged, baseURL)
|
||||
}
|
||||
|
||||
resolvedAPI := spec.API
|
||||
if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" {
|
||||
resolvedAPI = api.API(forced)
|
||||
|
|
@ -346,3 +367,84 @@ func (r *Resolved) Capabilities() Capabilities {
|
|||
caps.ThinkingLevels = r.ThinkingLevels.SupportedLevels()
|
||||
return caps
|
||||
}
|
||||
|
||||
// resolveRerank merges the rerank layers. Rerank has one settings struct
|
||||
// rather than one per protocol, so the model entry's compat object is decoded
|
||||
// unconditionally instead of being matched against a protocol.
|
||||
func resolveRerank(
|
||||
ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string,
|
||||
) (*Resolved, error) {
|
||||
protocol := vendor.RerankAPI
|
||||
if protocol == "" {
|
||||
protocol = api.RerankCohere
|
||||
}
|
||||
if !protocol.Known() {
|
||||
return nil, fmt.Errorf("catalog: unknown rerank api %q for provider %s", protocol, vendor.ID)
|
||||
}
|
||||
|
||||
// Lowest precedence first: protocol default, vendor, model entry, row spec
|
||||
// override. extra_config is the operator speaking about this one row, so it
|
||||
// comes last.
|
||||
settings := DefaultRerank()
|
||||
apply(&settings, &vendor.Compat.Rerank)
|
||||
if len(spec.Compat) > 0 {
|
||||
overlay := &RerankCompat{}
|
||||
if err := decodeCompat(spec.Compat, overlay); err != nil {
|
||||
return nil, fmt.Errorf("rerank compat: %w", err)
|
||||
}
|
||||
apply(&settings, overlay)
|
||||
}
|
||||
if raw := ref.Override.CompatJSON(); len(raw) > 0 {
|
||||
overlay := &RerankCompat{}
|
||||
if err := decodeCompat(raw, overlay); err != nil {
|
||||
return nil, fmt.Errorf("rerank compat: %w", err)
|
||||
}
|
||||
apply(&settings, overlay)
|
||||
}
|
||||
|
||||
if raw := strings.TrimSpace(ref.Extra[ExtraScoreScale]); raw != "" {
|
||||
scale := api.ScoreScale(strings.ToLower(raw))
|
||||
if scale != api.ScoreProbability && scale != api.ScoreLogit {
|
||||
return nil, fmt.Errorf(
|
||||
"catalog: invalid %s in extra_config: %q (expected %q or %q)",
|
||||
ExtraScoreScale, raw, api.ScoreProbability, api.ScoreLogit,
|
||||
)
|
||||
}
|
||||
settings.ScoreScale = scale
|
||||
}
|
||||
if raw := strings.TrimSpace(ref.Extra[ExtraTruncatePromptTokens]); raw != "" {
|
||||
if !settings.AcceptsTruncatePromptTokens {
|
||||
return nil, fmt.Errorf(
|
||||
"catalog: %s is a vLLM extension and %s does not implement it; "+
|
||||
"remove it from extra_config (it is accepted by self-hosted runtimes only)",
|
||||
ExtraTruncatePromptTokens, vendor.ID,
|
||||
)
|
||||
}
|
||||
budget, err := strconv.Atoi(raw)
|
||||
if err != nil || budget <= 0 {
|
||||
return nil, fmt.Errorf("catalog: invalid %s in extra_config: %q", ExtraTruncatePromptTokens, raw)
|
||||
}
|
||||
settings.TruncatePromptTokens = budget
|
||||
}
|
||||
// Checked after every layer: a model entry is what declares a dialect
|
||||
// this build cannot speak.
|
||||
if settings.UnsupportedReason != "" {
|
||||
return nil, fmt.Errorf(
|
||||
"catalog: %s does not serve %q through a protocol this build implements: %s",
|
||||
vendor.ID, spec.ID, settings.UnsupportedReason,
|
||||
)
|
||||
}
|
||||
out := &Resolved{
|
||||
Vendor: vendor,
|
||||
Spec: spec,
|
||||
Cataloged: cataloged,
|
||||
BaseURL: baseURL,
|
||||
RemoteModel: ref.Model,
|
||||
RerankAPI: protocol,
|
||||
Rerank: settings,
|
||||
}
|
||||
if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" {
|
||||
out.RemoteModel = override
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,14 +42,16 @@ const (
|
|||
// ExtraField describes one vendor-specific configuration input the model
|
||||
// editor renders dynamically (Azure api-version, LKEAP secret key, ...).
|
||||
type ExtraField struct {
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Type string `json:"type"` // "string", "number", "boolean", "select", "password"
|
||||
Required bool `json:"required"`
|
||||
Default string `json:"default,omitempty"`
|
||||
Placeholder string `json:"placeholder,omitempty"`
|
||||
Options []ExtraFieldOption `json:"options,omitempty"`
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Type string `json:"type"` // "string", "number", "boolean", "select", "password"
|
||||
Required bool `json:"required"`
|
||||
Default string `json:"default,omitempty"`
|
||||
Placeholder string `json:"placeholder,omitempty"`
|
||||
// Placeholders carries localized variants of Placeholder.
|
||||
Placeholders map[string]string `json:"placeholders,omitempty"`
|
||||
Options []ExtraFieldOption `json:"options,omitempty"`
|
||||
// ModelTypes restricts the field to some model types; empty means all.
|
||||
ModelTypes []types.ModelType `json:"model_types,omitempty"`
|
||||
// Secret marks values that must never be echoed back to the UI.
|
||||
|
|
@ -102,6 +104,11 @@ func localizedOr(table map[string]string, locale, fallback string) string {
|
|||
return fallback
|
||||
}
|
||||
|
||||
// LocalizedPlaceholder resolves the placeholder for a locale.
|
||||
func (f ExtraField) LocalizedPlaceholder(locale string) string {
|
||||
return localizedOr(f.Placeholders, locale, f.Placeholder)
|
||||
}
|
||||
|
||||
// CredentialLabelFor returns the credential naming for a model type, or nil
|
||||
// when the vendor uses the generic API-key wording.
|
||||
func (v *Vendor) CredentialLabelFor(modelType types.ModelType) *CredentialLabel {
|
||||
|
|
@ -119,10 +126,19 @@ func (v *Vendor) CredentialLabelFor(modelType types.ModelType) *CredentialLabel
|
|||
return nil
|
||||
}
|
||||
|
||||
// ExtraFieldOption is one choice of a select field.
|
||||
// ExtraFieldOption is one choice of a select field. Labels carries localized
|
||||
// variants keyed by locale, like every other operator-facing string here: an
|
||||
// option whose label is prose rather than an identifier is unreadable to half
|
||||
// the product without it.
|
||||
type ExtraFieldOption struct {
|
||||
Label string `json:"label"`
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// LocalizedLabel resolves the option label for a locale.
|
||||
func (o ExtraFieldOption) LocalizedLabel(locale string) string {
|
||||
return localizedOr(o.Labels, locale, o.Label)
|
||||
}
|
||||
|
||||
// ModelCost is priced per million tokens, in USD unless Currency says otherwise.
|
||||
|
|
@ -223,6 +239,9 @@ type Vendor struct {
|
|||
Icon []byte
|
||||
// API is the default chat protocol; a ModelSpec may override it.
|
||||
API api.API
|
||||
// RerankAPI is the rerank protocol. Register defaults it to the Cohere
|
||||
// shape for any vendor that serves rerank without naming another.
|
||||
RerankAPI api.RerankAPI
|
||||
// DefaultBaseURLs by model type; GetDefaultURL falls back to chat.
|
||||
DefaultBaseURLs map[types.ModelType]string
|
||||
ModelTypes []types.ModelType
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
package parity
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
|
@ -523,6 +524,13 @@ func TestCatalogInvariants(t *testing.T) {
|
|||
// Resolve validates the compat object against the model's
|
||||
// protocol: an unknown or misspelled key fails here.
|
||||
_, err := catalog.Resolve(catalog.Ref{Provider: v.ID, Model: m.ID, ModelType: m.Type})
|
||||
if bytes.Contains(m.Compat, []byte("unsupported_reason")) {
|
||||
// An entry that declares itself unserveable must say so
|
||||
// by refusing, not by resolving into a request shaped
|
||||
// for a protocol this build does not speak.
|
||||
assert.Error(t, err, "%s: declares unsupported_reason but still resolves", label)
|
||||
continue
|
||||
}
|
||||
assert.NoError(t, err, "%s: does not resolve", label)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,522 @@
|
|||
package parity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestEveryRerankVendorResolvesToAKnownProtocol walks the catalog, so a newly
|
||||
// added vendor is covered without touching this file.
|
||||
func TestEveryRerankVendorResolvesToAKnownProtocol(t *testing.T) {
|
||||
vendors := catalog.ListByType(types.ModelTypeRerank)
|
||||
require.NotEmpty(t, vendors, "no vendor serves rerank")
|
||||
|
||||
for _, v := range vendors {
|
||||
t.Run(v.ID, func(t *testing.T) {
|
||||
model := "some-rerank-model"
|
||||
if catalogued := v.ModelsByType(types.ModelTypeRerank); len(catalogued) > 0 {
|
||||
model = catalogued[0].ID
|
||||
}
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: v.ID, Model: model, ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, resolved.RerankAPI.Known(),
|
||||
"unknown rerank protocol %q", resolved.RerankAPI)
|
||||
// The catch-all vendor has no endpoint of its own: the operator
|
||||
// supplies one, and Validate requires it.
|
||||
if v.ID != catalog.GenericID {
|
||||
assert.NotEmpty(t, resolved.BaseURL, "rerank vendors need a default endpoint")
|
||||
}
|
||||
|
||||
// A vendor that declares a ceiling must declare a usable one.
|
||||
assert.GreaterOrEqual(t, resolved.Rerank.MaxDocuments, 0)
|
||||
assert.GreaterOrEqual(t, resolved.Rerank.MaxDocumentChars, 0)
|
||||
if resolved.Rerank.ScoreScale != "" {
|
||||
assert.Contains(t,
|
||||
[]api.ScoreScale{api.ScoreProbability, api.ScoreLogit},
|
||||
resolved.Rerank.ScoreScale)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankProtocolAssignment pins which dialect each vendor speaks. The
|
||||
// wire shapes are mutually incompatible, so a silent reassignment breaks
|
||||
// every rerank call for that vendor.
|
||||
func TestRerankProtocolAssignment(t *testing.T) {
|
||||
for id, want := range map[string]api.RerankAPI{
|
||||
"aliyun": api.RerankDashScope,
|
||||
"nvidia": api.RerankNIM,
|
||||
"lkeap": api.RerankTencentLKEAP,
|
||||
"volcengine": api.RerankVolcengineKnowledge,
|
||||
"zhipu": api.RerankCohere,
|
||||
"jina": api.RerankCohere,
|
||||
"siliconflow": api.RerankCohere,
|
||||
"qianfan": api.RerankCohere,
|
||||
"gpustack": api.RerankCohere,
|
||||
"generic": api.RerankCohere,
|
||||
"openai": api.RerankCohere,
|
||||
"weknoracloud": api.RerankCohere,
|
||||
"novita": api.RerankCohere,
|
||||
"openrouter": api.RerankCohere,
|
||||
"litellm": api.RerankCohere,
|
||||
} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, resolved.RerankAPI)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankOutboundShapePerProtocol builds the real client through
|
||||
// rerank.NewReranker and asserts what each dialect puts on the wire.
|
||||
func TestRerankOutboundShapePerProtocol(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
provider string
|
||||
model string
|
||||
assert func(t *testing.T, path string, body map[string]any)
|
||||
reply string
|
||||
}{
|
||||
{
|
||||
name: "cohere shape", provider: "siliconflow", model: "BAAI/bge-reranker-v2-m3",
|
||||
reply: `{"results":[{"index":0,"relevance_score":0.5}]}`,
|
||||
assert: func(t *testing.T, path string, body map[string]any) {
|
||||
assert.Equal(t, "/v1/rerank", path)
|
||||
assert.Equal(t, "BAAI/bge-reranker-v2-m3", body["model"])
|
||||
assert.Equal(t, "q", body["query"])
|
||||
assert.Equal(t, []any{"d0"}, body["documents"])
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dashscope shape", provider: "aliyun", model: "gte-rerank-v2",
|
||||
reply: `{"output":{"results":[{"index":0,"relevance_score":0.5,"document":{"text":"d0"}}]}}`,
|
||||
assert: func(t *testing.T, _ string, body map[string]any) {
|
||||
input, ok := body["input"].(map[string]any)
|
||||
require.True(t, ok, "DashScope wraps the query and documents in input")
|
||||
assert.Equal(t, "q", input["query"])
|
||||
assert.Contains(t, body, "parameters")
|
||||
assert.NotContains(t, body, "documents", "documents live under input")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nim shape", provider: "nvidia", model: "nvidia/nv-rerankqa-mistral-4b-v3",
|
||||
reply: `{"rankings":[{"index":0,"logit":0.0}]}`,
|
||||
assert: func(t *testing.T, _ string, body map[string]any) {
|
||||
query, ok := body["query"].(map[string]any)
|
||||
require.True(t, ok, "NIM sends the query as an object")
|
||||
assert.Equal(t, "q", query["text"])
|
||||
assert.Equal(t, []any{map[string]any{"text": "d0"}}, body["passages"])
|
||||
assert.Equal(t, "END", body["truncate"], "NIM fails over-long input unless told to cut")
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(tc.reply))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
reranker, err := rerank.NewReranker(&rerank.RerankerConfig{
|
||||
Provider: tc.provider, ModelName: tc.model, APIKey: "k",
|
||||
BaseURL: server.URL + rerankPathFor(t, tc.provider),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = reranker.Rerank(context.Background(), "q", []string{"d0"})
|
||||
require.NoError(t, err)
|
||||
tc.assert(t, gotPath, gotBody)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// rerankPathFor keeps the loopback base URL shaped like the vendor's own, so
|
||||
// the test exercises the same path handling production does.
|
||||
func rerankPathFor(t *testing.T, provider string) string {
|
||||
t.Helper()
|
||||
switch provider {
|
||||
case "aliyun":
|
||||
return "/api/v1/services/rerank/text-rerank/text-rerank"
|
||||
case "nvidia":
|
||||
return "/v1/retrieval/nvidia/reranking"
|
||||
default:
|
||||
return "/v1"
|
||||
}
|
||||
}
|
||||
|
||||
// TestNvidiaRerankScoresAreComparable is the end-to-end form of the unit test
|
||||
// in the rerank package: a NIM logit must reach the retrieval pipeline on the
|
||||
// same 0..1 scale as every other vendor, because they all meet one
|
||||
// RerankThreshold.
|
||||
func TestNvidiaRerankScoresAreComparable(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"rankings":[{"index":0,"logit":-1.171875},{"index":1,"logit":4.0}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
reranker, err := rerank.NewReranker(&rerank.RerankerConfig{
|
||||
Provider: "nvidia", ModelName: "nvidia/nv-rerankqa-mistral-4b-v3", APIKey: "k",
|
||||
BaseURL: server.URL + "/v1/retrieval/nvidia/reranking",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := reranker.Rerank(context.Background(), "q", []string{"d0", "d1"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
for _, item := range got {
|
||||
assert.GreaterOrEqual(t, item.RelevanceScore, 0.0)
|
||||
assert.LessOrEqual(t, item.RelevanceScore, 1.0)
|
||||
}
|
||||
// Results come back ranked, so the logit of 4.0 leads and the negative
|
||||
// one follows: the conversion is monotonic and the ranking is preserved.
|
||||
assert.Equal(t, 1, got[0].Index)
|
||||
assert.Greater(t, got[0].RelevanceScore, got[1].RelevanceScore)
|
||||
}
|
||||
|
||||
// TestSignedRerankVendorRequiresItsIdentityPair keeps the pre-catalog
|
||||
// contract: WeKnora Cloud rerank is signed with an AppID / AppSecret pair, and
|
||||
// a row without one must fail at construction rather than send unsigned
|
||||
// requests that the far end rejects with something less readable.
|
||||
func TestSignedRerankVendorRequiresItsIdentityPair(t *testing.T) {
|
||||
_, err := rerank.NewReranker(&rerank.RerankerConfig{
|
||||
Provider: "weknoracloud", ModelName: "rerank",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "AppID")
|
||||
|
||||
_, err = rerank.NewReranker(&rerank.RerankerConfig{
|
||||
Provider: "weknoracloud", ModelName: "rerank", AppID: "app",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "AppSecret")
|
||||
}
|
||||
|
||||
// TestTruncatePromptTokensOnlyReachesVLLMClassVendors pins the gate on the
|
||||
// vLLM extension restored from the pre-catalog client. It is opt-in per row,
|
||||
// but the row can only opt into it on a runtime that implements it: no
|
||||
// managed vendor documents the field, and sending it to one is how an
|
||||
// undocumented parameter ends up on every request.
|
||||
func TestTruncatePromptTokensOnlyReachesVLLMClassVendors(t *testing.T) {
|
||||
optIn := map[string]string{catalog.ExtraTruncatePromptTokens: "512"}
|
||||
|
||||
for _, id := range []string{"generic", "gpustack"} {
|
||||
t.Run(id+" accepts it", func(t *testing.T) {
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
BaseURL: "http://127.0.0.1:9/v1", Extra: optIn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 512, resolved.Rerank.TruncatePromptTokens)
|
||||
})
|
||||
}
|
||||
|
||||
for _, id := range []string{"jina", "zhipu", "siliconflow", "qianfan", "weknoracloud", "aliyun", "nvidia"} {
|
||||
t.Run(id+" rejects it", func(t *testing.T) {
|
||||
_, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: id, Model: "m", ModelType: types.ModelTypeRerank, Extra: optIn,
|
||||
})
|
||||
require.Error(t, err, "a managed vendor must not silently accept a vLLM extension")
|
||||
assert.Contains(t, err.Error(), "vLLM extension")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncatePromptTokensRejectsAnInvalidValue(t *testing.T) {
|
||||
for _, raw := range []string{"0", "-1", "abc"} {
|
||||
_, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: "generic", Model: "m", ModelType: types.ModelTypeRerank,
|
||||
BaseURL: "http://127.0.0.1:9/v1",
|
||||
Extra: map[string]string{catalog.ExtraTruncatePromptTokens: raw},
|
||||
})
|
||||
require.Error(t, err, "value %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankCeilingsAreTheDocumentedOnes pins the numbers the shared batching
|
||||
// layer enforces. They used to live inside each client, where a dedicated
|
||||
// test watched them; now they are declarations, so they need their own guard —
|
||||
// TestEveryRerankVendorResolvesToAKnownProtocol only checks they are sane.
|
||||
func TestRerankCeilingsAreTheDocumentedOnes(t *testing.T) {
|
||||
for id, want := range map[string]catalog.RerankSettings{
|
||||
// docs.bigmodel.cn: 最多 128 条,query 与单条文档各 4096 字符
|
||||
"zhipu": {MaxDocuments: 128, MaxQueryChars: 4096, MaxDocumentChars: 4096},
|
||||
// cloud.tencent.com/document/product/1772: RunRerank 60 docs,
|
||||
// Query + Docs together 2000 characters, one request at a time.
|
||||
"lkeap": {MaxDocuments: 60, MaxRequestChars: 2000, MaxConcurrency: 1},
|
||||
// Ark Knowledge Service rerank: 50 per request.
|
||||
"volcengine": {MaxDocuments: 50, MaxConcurrency: 4},
|
||||
// NIM reranking: passages is capped at 512 items.
|
||||
"nvidia": {MaxDocuments: 512},
|
||||
// cloud.baidu.com/doc/qianfan-api: 文本数量不超过64, query 不超过
|
||||
// 1600 个字符, 每条 document 不超过 4096 个字符.
|
||||
"qianfan": {MaxDocuments: 64, MaxQueryChars: 1600, MaxDocumentChars: 4096},
|
||||
// help.aliyun.com text-rerank: 500 documents per request. Its length
|
||||
// limits are stated in tokens, which runes cannot express.
|
||||
"aliyun": {MaxDocuments: 500},
|
||||
} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
got := resolved.Rerank
|
||||
assert.Equal(t, want.MaxDocuments, got.MaxDocuments, "max documents")
|
||||
assert.Equal(t, want.MaxQueryChars, got.MaxQueryChars, "max query characters")
|
||||
assert.Equal(t, want.MaxDocumentChars, got.MaxDocumentChars, "max document characters")
|
||||
assert.Equal(t, want.MaxRequestChars, got.MaxRequestChars, "max request characters")
|
||||
assert.Equal(t, want.MaxConcurrency, got.MaxConcurrency, "max concurrency")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWeKnoraCloudRerankKeepsItsDeadline pins the one rerank vendor that has
|
||||
// ever had a client-level timeout. Without it a hung endpoint holds the
|
||||
// retrieval stage open for as long as the caller's context allows.
|
||||
func TestWeKnoraCloudRerankKeepsItsDeadline(t *testing.T) {
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: "weknoracloud", Model: "rerank", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 60, resolved.Rerank.RequestTimeout)
|
||||
|
||||
// Everyone else has always run without one and relies on the caller.
|
||||
for _, id := range []string{"zhipu", "jina", "siliconflow", "aliyun", "nvidia", "generic"} {
|
||||
other, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: id, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, other.Rerank.RequestTimeout, "%s did not have a client deadline before", id)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTruncatePromptTokensIsReachableFromTheEditor closes the loop between the
|
||||
// vendor accepting the extension and an operator being able to turn it on.
|
||||
// Before the catalog it was readable from extra_config but had no input in the
|
||||
// model editor, so the only way to set it was the API or the database.
|
||||
// GET /models/providers renders extra fields dynamically, so declaring one is
|
||||
// all it takes — and it must be declared exactly where the extension is
|
||||
// accepted, or the form offers a switch the vendor will reject.
|
||||
func TestTruncatePromptTokensIsReachableFromTheEditor(t *testing.T) {
|
||||
for _, v := range catalog.ListByType(types.ModelTypeRerank) {
|
||||
var field *catalog.ExtraField
|
||||
for i := range v.ExtraFields {
|
||||
if v.ExtraFields[i].Key == catalog.ExtraTruncatePromptTokens {
|
||||
field = &v.ExtraFields[i]
|
||||
}
|
||||
}
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: v.ID, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
if !resolved.Rerank.AcceptsTruncatePromptTokens {
|
||||
assert.Nil(t, field, "%s does not accept the extension, so it must not offer the input", v.ID)
|
||||
continue
|
||||
}
|
||||
require.NotNil(t, field, "%s accepts the extension but offers no way to set it", v.ID)
|
||||
assert.Equal(t, "number", field.Type)
|
||||
assert.False(t, field.Required, "the extension is opt-in")
|
||||
assert.Equal(t, []types.ModelType{types.ModelTypeRerank}, field.ModelTypes,
|
||||
"the input belongs to the rerank form only")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRerankScoreScalesMatchTheVendorDocs pins which vendors return something
|
||||
// other than a 0..1 relevance score. Getting this wrong is silent: the number
|
||||
// still looks like a score, and the retrieval threshold still compares it, so
|
||||
// a logit-scaled vendor simply loses every negatively scored document.
|
||||
func TestRerankScoreScalesMatchTheVendorDocs(t *testing.T) {
|
||||
// Both spell the field like a probability and return neither.
|
||||
// NIM: rankings[].logit, e.g. 0.226 / -1.171 / -6.31
|
||||
// GPUStack: relevance_score, e.g. 1.951 / -3.734 / -6.157
|
||||
logitScaled := map[string]bool{"nvidia": true, "gpustack": true}
|
||||
|
||||
for _, v := range catalog.ListByType(types.ModelTypeRerank) {
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: v.ID, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
want := api.ScoreProbability
|
||||
if logitScaled[v.ID] {
|
||||
want = api.ScoreLogit
|
||||
}
|
||||
assert.Equal(t, want, resolved.Rerank.ScoreScale, "%s score scale", v.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnimplementedRerankDialectsAreRefused covers a model that names a
|
||||
// protocol no package implements. Hiding it from the picker stops new rows;
|
||||
// a row that already names it has to fail somewhere, and failing at
|
||||
// construction with the reason beats sending a request shaped for the wrong
|
||||
// protocol and reporting whatever the decoder makes of the reply.
|
||||
func TestUnimplementedRerankDialectsAreRefused(t *testing.T) {
|
||||
v, ok := catalog.Get("aliyun")
|
||||
require.True(t, ok)
|
||||
|
||||
offered := make([]string, 0)
|
||||
for _, m := range v.ModelsByType(types.ModelTypeRerank) {
|
||||
offered = append(offered, m.ID)
|
||||
}
|
||||
assert.NotContains(t, offered, "qwen3-rerank", "the picker must not offer an unimplemented dialect")
|
||||
assert.Contains(t, offered, "gte-rerank-v2")
|
||||
|
||||
_, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: "aliyun", Model: "qwen3-rerank", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "/compatible-api/v1/reranks",
|
||||
"the error should name the protocol the model actually speaks")
|
||||
|
||||
// The vendor's other rerank model is unaffected.
|
||||
_, err = catalog.Resolve(catalog.Ref{
|
||||
Provider: "aliyun", Model: "gte-rerank-v2", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestGatewayRerankEndpoints pins the URL each gateway's rerank rows reach.
|
||||
// All three serve the Cohere dialect the protocol package already speaks, so
|
||||
// opening the type was a declaration — but LiteLLM documents the route on the
|
||||
// proxy root rather than under /v1, which is why it needs its own default.
|
||||
func TestGatewayRerankEndpoints(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
provider string
|
||||
wantURL string
|
||||
}{
|
||||
// https://docs.novita.ai/api-reference/model-apis-llm-create-rerank
|
||||
{provider: "novita", wantURL: "https://api.novita.ai/openai/v1/rerank"},
|
||||
// https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request
|
||||
{provider: "openrouter", wantURL: "https://openrouter.ai/api/v1/rerank"},
|
||||
// https://docs.litellm.ai/docs/rerank — the curl example posts to
|
||||
// http://0.0.0.0:4000/rerank, not /v1/rerank.
|
||||
{provider: "litellm", wantURL: "http://your_litellm_proxy/rerank"},
|
||||
} {
|
||||
t.Run(tc.provider, func(t *testing.T) {
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: tc.provider, Model: "m", ModelType: types.ModelTypeRerank,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, api.RerankCohere, resolved.RerankAPI)
|
||||
|
||||
// Rebuild the URL the way cohererank does.
|
||||
got := strings.TrimRight(resolved.BaseURL, "/")
|
||||
if !strings.HasSuffix(got, "/rerank") {
|
||||
got += "/rerank"
|
||||
}
|
||||
assert.Equal(t, tc.wantURL, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestScoreScaleIsOverridablePerRow covers the gateways, where the vendor
|
||||
// cannot know the answer: GPUStack and a generic endpoint serve whatever
|
||||
// reranker was deployed behind them, and the two families disagree — BGE
|
||||
// answers a 0..1 probability, Qwen3-Reranker an unbounded score. Converting a
|
||||
// probability as if it were a logit squeezes every score into [0.5, 0.73],
|
||||
// which makes a relevance threshold meaningless, so the operator needs a way
|
||||
// to say which one is actually there.
|
||||
func TestScoreScaleIsOverridablePerRow(t *testing.T) {
|
||||
for _, id := range []string{"gpustack", "generic"} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
base := "http://127.0.0.1:9/v1"
|
||||
overridden, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: id, Model: "bge-reranker-v2-m3", ModelType: types.ModelTypeRerank,
|
||||
BaseURL: base, Extra: map[string]string{catalog.ExtraScoreScale: "probability"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, api.ScoreProbability, overridden.Rerank.ScoreScale)
|
||||
|
||||
// And the input to set it is rendered by the editor.
|
||||
v, ok := catalog.Get(id)
|
||||
require.True(t, ok)
|
||||
var field *catalog.ExtraField
|
||||
for i := range v.ExtraFields {
|
||||
if v.ExtraFields[i].Key == catalog.ExtraScoreScale {
|
||||
field = &v.ExtraFields[i]
|
||||
}
|
||||
}
|
||||
require.NotNil(t, field, "%s must offer the override it needs", id)
|
||||
assert.Equal(t, "select", field.Type)
|
||||
assert.Len(t, field.Options, 2)
|
||||
})
|
||||
}
|
||||
|
||||
_, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: "gpustack", Model: "m", ModelType: types.ModelTypeRerank,
|
||||
BaseURL: "http://127.0.0.1:9/v1",
|
||||
Extra: map[string]string{catalog.ExtraScoreScale: "sigmoid"},
|
||||
})
|
||||
require.Error(t, err, "an unknown scale must not be treated as a probability")
|
||||
}
|
||||
|
||||
// TestGatewayVendorsOfferTheScoreScaleOverride keeps the override where the
|
||||
// question is open. A gateway cannot know which reranker is deployed behind
|
||||
// it, so all three offer the input; the other vendors' scales are settled by
|
||||
// their own documentation and must not offer a switch that only lets an
|
||||
// operator get it wrong.
|
||||
func TestGatewayVendorsOfferTheScoreScaleOverride(t *testing.T) {
|
||||
gateways := map[string]bool{"generic": true, "gpustack": true, "litellm": true}
|
||||
|
||||
for _, v := range catalog.ListByType(types.ModelTypeRerank) {
|
||||
var field *catalog.ExtraField
|
||||
for i := range v.ExtraFields {
|
||||
if v.ExtraFields[i].Key == catalog.ExtraScoreScale {
|
||||
field = &v.ExtraFields[i]
|
||||
}
|
||||
}
|
||||
if !gateways[v.ID] {
|
||||
assert.Nil(t, field, "%s: its documentation settles the scale", v.ID)
|
||||
continue
|
||||
}
|
||||
require.NotNil(t, field, "%s is a gateway and needs the override", v.ID)
|
||||
require.Len(t, field.Options, 2)
|
||||
for _, opt := range field.Options {
|
||||
assert.NotEmpty(t, opt.LocalizedLabel("zh-CN"), "%s: option %q has no label", v.ID, opt.Value)
|
||||
assert.NotEqual(t, opt.Label, opt.LocalizedLabel("zh-CN"),
|
||||
"%s: option %q is prose and needs a zh-CN label", v.ID, opt.Value)
|
||||
}
|
||||
assert.NotEqual(t, field.Placeholder, field.LocalizedPlaceholder("zh-CN"),
|
||||
"%s: the placeholder is prose and needs a zh-CN variant", v.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Every operator-facing string a vendor declares has to exist in both
|
||||
// languages, or half the product reads English prose in a Chinese form.
|
||||
func TestEveryExtraFieldProseIsLocalized(t *testing.T) {
|
||||
for _, v := range catalog.List() {
|
||||
for _, field := range v.ExtraFields {
|
||||
if field.Placeholder != "" && strings.Contains(field.Placeholder, " ") {
|
||||
assert.NotEqual(t, field.Placeholder, field.LocalizedPlaceholder("zh-CN"),
|
||||
"%s/%s: placeholder is prose without a zh-CN variant", v.ID, field.Key)
|
||||
}
|
||||
for _, opt := range field.Options {
|
||||
// An identifier (a region code, a version) is the same in
|
||||
// every language; prose is not.
|
||||
if strings.Contains(opt.Label, " ") {
|
||||
assert.NotEqual(t, opt.Label, opt.LocalizedLabel("zh-CN"),
|
||||
"%s/%s: option %q is prose without a zh-CN label", v.ID, field.Key, opt.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// AliyunReranker implements a reranking system based on Aliyun DashScope models
|
||||
type AliyunReranker struct {
|
||||
modelName string // Name of the model used for reranking
|
||||
modelID string // Unique identifier of the model
|
||||
apiKey string // API key for authentication
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
customHeaders map[string]string
|
||||
}
|
||||
|
||||
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
func (r *AliyunReranker) SetCustomHeaders(headers map[string]string) {
|
||||
r.customHeaders = headers
|
||||
}
|
||||
|
||||
// AliyunRerankRequest represents a request to rerank documents using Aliyun DashScope API
|
||||
type AliyunRerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Input AliyunRerankInput `json:"input"` // Input containing query and documents
|
||||
Parameters AliyunRerankParameters `json:"parameters"` // Parameters for the reranking
|
||||
}
|
||||
|
||||
// AliyunRerankInput contains the query and documents for reranking
|
||||
type AliyunRerankInput struct {
|
||||
Query string `json:"query"` // Query text to compare documents against
|
||||
Documents []string `json:"documents"` // List of document texts to rerank
|
||||
}
|
||||
|
||||
// AliyunRerankParameters contains parameters for the reranking request
|
||||
type AliyunRerankParameters struct {
|
||||
ReturnDocuments bool `json:"return_documents"` // Whether to return documents in response
|
||||
TopN int `json:"top_n"` // Number of top results to return
|
||||
}
|
||||
|
||||
// AliyunRerankResponse represents the response from Aliyun DashScope reranking request
|
||||
type AliyunRerankResponse struct {
|
||||
Output AliyunOutput `json:"output"` // Output containing results
|
||||
Usage AliyunUsage `json:"usage"` // Token usage information
|
||||
}
|
||||
|
||||
// AliyunOutput contains the reranking results
|
||||
type AliyunOutput struct {
|
||||
Results []AliyunRankResult `json:"results"` // Ranked results with relevance scores
|
||||
}
|
||||
|
||||
// AliyunRankResult represents a single reranking result from Aliyun
|
||||
type AliyunRankResult struct {
|
||||
Document AliyunDocument `json:"document"` // Document information
|
||||
Index int `json:"index"` // Original index of the document
|
||||
RelevanceScore float64 `json:"relevance_score"` // Relevance score
|
||||
}
|
||||
|
||||
// AliyunDocument represents document information in Aliyun response
|
||||
type AliyunDocument struct {
|
||||
Text string `json:"text"` // Document text
|
||||
}
|
||||
|
||||
// AliyunUsage contains information about token usage in the Aliyun API request
|
||||
type AliyunUsage struct {
|
||||
TotalTokens int `json:"total_tokens"` // Total tokens consumed
|
||||
}
|
||||
|
||||
// NewAliyunReranker creates a new instance of Aliyun reranker with the provided configuration
|
||||
func NewAliyunReranker(config *RerankerConfig) (*AliyunReranker, error) {
|
||||
apiKey := config.APIKey
|
||||
baseURL := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
|
||||
if url := config.BaseURL; url != "" {
|
||||
baseURL = url
|
||||
}
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &AliyunReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query using Aliyun DashScope API
|
||||
func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body
|
||||
requestBody := &AliyunRerankRequest{
|
||||
Model: r.modelName,
|
||||
Input: AliyunRerankInput{
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
},
|
||||
Parameters: AliyunRerankParameters{
|
||||
ReturnDocuments: true,
|
||||
TopN: len(documents), // Return all documents
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
// Send the request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
secutils.ApplyCustomHeaders(req, r.customHeaders)
|
||||
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("aliyun rerank API error: Http Status: %s, Body: %s", resp.Status, string(body))
|
||||
}
|
||||
|
||||
var response AliyunRerankResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
// Convert Aliyun results to standard RankResult format
|
||||
results := make([]RankResult, len(response.Output.Results))
|
||||
for i, aliyunResult := range response.Output.Results {
|
||||
results[i] = RankResult{
|
||||
Index: aliyunResult.Index,
|
||||
Document: DocumentInfo{
|
||||
Text: aliyunResult.Document.Text,
|
||||
},
|
||||
RelevanceScore: aliyunResult.RelevanceScore,
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the name of the reranking model
|
||||
func (r *AliyunReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the unique identifier of the reranking model
|
||||
func (r *AliyunReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// JinaReranker implements a reranking system using Jina AI API
|
||||
// Jina API uses different parameters than standard OpenAI-compatible APIs
|
||||
type JinaReranker struct {
|
||||
modelName string // Name of the model used for reranking
|
||||
modelID string // Unique identifier of the model
|
||||
apiKey string // API key for authentication
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
customHeaders map[string]string
|
||||
}
|
||||
|
||||
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
func (r *JinaReranker) SetCustomHeaders(headers map[string]string) {
|
||||
r.customHeaders = headers
|
||||
}
|
||||
|
||||
// JinaRerankRequest represents a Jina rerank request
|
||||
// Note: Jina does NOT support truncate_prompt_tokens parameter
|
||||
type JinaRerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query string `json:"query"` // Query text to compare documents against
|
||||
Documents []string `json:"documents"` // List of document texts to rerank
|
||||
TopN int `json:"top_n,omitempty"` // Number of top results to return
|
||||
ReturnDocuments bool `json:"return_documents,omitempty"` // Whether to return document text in response
|
||||
}
|
||||
|
||||
// JinaRerankResponse represents the response from a Jina reranking request
|
||||
type JinaRerankResponse struct {
|
||||
Model string `json:"model"` // Model used for reranking
|
||||
Results []RankResult `json:"results"` // Ranked results with relevance scores
|
||||
Usage struct {
|
||||
TotalTokens int `json:"total_tokens"` // Total tokens consumed
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
// NewJinaReranker creates a new instance of Jina reranker with the provided configuration
|
||||
func NewJinaReranker(config *RerankerConfig) (*JinaReranker, error) {
|
||||
apiKey := config.APIKey
|
||||
baseURL := "https://api.jina.ai/v1"
|
||||
if url := config.BaseURL; url != "" {
|
||||
baseURL = url
|
||||
}
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &JinaReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query
|
||||
func (r *JinaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body - Jina does NOT use truncate_prompt_tokens
|
||||
requestBody := &JinaRerankRequest{
|
||||
Model: r.modelName,
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
ReturnDocuments: true, // Return document text in response
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
// Send the request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
secutils.ApplyCustomHeaders(req, r.customHeaders)
|
||||
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body))
|
||||
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
|
||||
}
|
||||
|
||||
var response JinaRerankResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
return response.Results, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the name of the reranking model
|
||||
func (r *JinaReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the unique identifier of the reranking model
|
||||
func (r *JinaReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
|
|
@ -4,19 +4,15 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
|
||||
lkeap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lkeap/v20240522"
|
||||
)
|
||||
|
||||
const (
|
||||
// LKEAPMaxDocumentsPerRequest is the maximum number of documents accepted by RunRerank.
|
||||
LKEAPMaxDocumentsPerRequest = 60
|
||||
// LKEAPMaxRequestCharacters is the maximum combined length of Query and Docs accepted by RunRerank.
|
||||
LKEAPMaxRequestCharacters = 2000
|
||||
// LKEAPRerankEndpoint 腾讯云知识引擎原子能力 Rerank API 域名
|
||||
LKEAPRerankEndpoint = "lkeap.tencentcloudapi.com"
|
||||
// LKEAPDefaultRegion RunRerank 支持的地域,默认广州
|
||||
|
|
@ -25,16 +21,17 @@ const (
|
|||
LKEAPDefaultRerankModel = "lke-reranker-base"
|
||||
)
|
||||
|
||||
// LKEAPReranker 使用腾讯云知识引擎原子能力 RunRerank 接口进行重排序。
|
||||
// 鉴权使用腾讯云 API 密钥:APIKey 为 SecretId,AppSecret 为 SecretKey。
|
||||
type LKEAPReranker struct {
|
||||
// lkeapClient calls Tencent Cloud's RunRerank action through the official
|
||||
// SDK, which owns the TC3 signing this API requires. It implements
|
||||
// api.Reranker and nothing else: the per-request ceilings (60 documents,
|
||||
// 2000 characters for Query plus Docs together) are declared on the vendor
|
||||
// and enforced by protocolReranker, like every other vendor's.
|
||||
type lkeapClient struct {
|
||||
modelName string
|
||||
modelID string
|
||||
client *lkeap.Client
|
||||
}
|
||||
|
||||
// NewLKEAPReranker 创建 LKEAP rerank 客户端。
|
||||
func NewLKEAPReranker(config *RerankerConfig) (*LKEAPReranker, error) {
|
||||
func newLKEAPClient(config *RerankerConfig, resolved *catalog.Resolved) (api.Reranker, error) {
|
||||
secretID := strings.TrimSpace(config.APIKey)
|
||||
secretKey := strings.TrimSpace(config.AppSecret)
|
||||
if secretKey == "" && config.ExtraConfig != nil {
|
||||
|
|
@ -60,86 +57,23 @@ func NewLKEAPReranker(config *RerankerConfig) (*LKEAPReranker, error) {
|
|||
return nil, fmt.Errorf("create LKEAP client: %w", err)
|
||||
}
|
||||
|
||||
modelName := strings.TrimSpace(config.ModelName)
|
||||
modelName := strings.TrimSpace(resolved.RemoteModel)
|
||||
if modelName == "" {
|
||||
modelName = LKEAPDefaultRerankModel
|
||||
}
|
||||
|
||||
return &LKEAPReranker{
|
||||
modelName: modelName,
|
||||
modelID: config.ModelID,
|
||||
client: client,
|
||||
}, nil
|
||||
return &lkeapClient{modelName: modelName, client: client}, nil
|
||||
}
|
||||
|
||||
// Rerank 调用 RunRerank 对文档按与 query 的相关性打分。
|
||||
func (r *LKEAPReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
if len(documents) == 0 {
|
||||
return []RankResult{}, nil
|
||||
}
|
||||
|
||||
batches, err := lkeapRerankBatches(query, documents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results := make([]RankResult, 0, len(documents))
|
||||
for _, batch := range batches {
|
||||
batchResults, err := r.rerankBatch(ctx, query, batch.documents)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range batchResults {
|
||||
batchResults[i].Index += batch.start
|
||||
}
|
||||
results = append(results, batchResults...)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type lkeapRerankBatch struct {
|
||||
start int
|
||||
documents []string
|
||||
}
|
||||
|
||||
func lkeapRerankBatches(query string, documents []string) ([]lkeapRerankBatch, error) {
|
||||
queryLength := utf8.RuneCountInString(query)
|
||||
if queryLength >= LKEAPMaxRequestCharacters {
|
||||
return nil, fmt.Errorf("LKEAP rerank query is %d characters; Query and Docs together support at most %d characters", queryLength, LKEAPMaxRequestCharacters)
|
||||
}
|
||||
|
||||
batches := make([]lkeapRerankBatch, 0, (len(documents)+LKEAPMaxDocumentsPerRequest-1)/LKEAPMaxDocumentsPerRequest)
|
||||
batchStart := 0
|
||||
batchLength := queryLength
|
||||
batchDocuments := make([]string, 0, LKEAPMaxDocumentsPerRequest)
|
||||
for index, document := range documents {
|
||||
documentLength := utf8.RuneCountInString(document)
|
||||
if queryLength+documentLength > LKEAPMaxRequestCharacters {
|
||||
return nil, fmt.Errorf("LKEAP rerank document at index %d is %d characters; Query and each document together support at most %d characters", index, documentLength, LKEAPMaxRequestCharacters)
|
||||
}
|
||||
if len(batchDocuments) == LKEAPMaxDocumentsPerRequest || batchLength+documentLength > LKEAPMaxRequestCharacters {
|
||||
batches = append(batches, lkeapRerankBatch{start: batchStart, documents: batchDocuments})
|
||||
batchStart = index
|
||||
batchLength = queryLength
|
||||
batchDocuments = make([]string, 0, LKEAPMaxDocumentsPerRequest)
|
||||
}
|
||||
batchDocuments = append(batchDocuments, document)
|
||||
batchLength += documentLength
|
||||
}
|
||||
if len(batchDocuments) > 0 {
|
||||
batches = append(batches, lkeapRerankBatch{start: batchStart, documents: batchDocuments})
|
||||
}
|
||||
return batches, nil
|
||||
}
|
||||
|
||||
func (r *LKEAPReranker) rerankBatch(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Rerank scores one batch. RunRerank answers a bare score list in input
|
||||
// order rather than an indexed, sorted result set.
|
||||
func (r *lkeapClient) Rerank(
|
||||
ctx context.Context, query string, documents []string,
|
||||
) ([]api.RerankResult, error) {
|
||||
req := lkeap.NewRunRerankRequest()
|
||||
req.Query = common.StringPtr(query)
|
||||
req.Docs = common.StringPtrs(documents)
|
||||
req.Model = common.StringPtr(r.modelName)
|
||||
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, LKEAPRerankEndpoint, query, documents))
|
||||
|
||||
resp, err := r.client.RunRerankWithContext(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LKEAP RunRerank: %w", err)
|
||||
|
|
@ -147,35 +81,18 @@ func (r *LKEAPReranker) rerankBatch(ctx context.Context, query string, documents
|
|||
if resp == nil || resp.Response == nil || len(resp.Response.ScoreList) == 0 {
|
||||
return nil, fmt.Errorf("LKEAP rerank API returned empty score list")
|
||||
}
|
||||
|
||||
scores := resp.Response.ScoreList
|
||||
if len(scores) != len(documents) {
|
||||
return nil, fmt.Errorf("LKEAP rerank score count mismatch: got %d scores for %d documents",
|
||||
len(scores), len(documents))
|
||||
}
|
||||
|
||||
results := make([]RankResult, len(documents))
|
||||
results := make([]api.RerankResult, len(documents))
|
||||
for i, score := range scores {
|
||||
if score == nil {
|
||||
continue
|
||||
}
|
||||
results[i] = RankResult{
|
||||
Index: i,
|
||||
Document: DocumentInfo{
|
||||
Text: documents[i],
|
||||
},
|
||||
RelevanceScore: *score,
|
||||
results[i] = api.RerankResult{Index: i, Text: documents[i]}
|
||||
if score != nil {
|
||||
results[i].Score = *score
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the rerank model name.
|
||||
func (r *LKEAPReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the model ID.
|
||||
func (r *LKEAPReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,178 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
|
||||
lkeap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/lkeap/v20240522"
|
||||
)
|
||||
|
||||
func TestNewLKEAPReranker_requiresCredentials(t *testing.T) {
|
||||
_, err := NewLKEAPReranker(&RerankerConfig{
|
||||
ModelName: "lke-reranker-base",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "secret_id")
|
||||
}
|
||||
|
||||
func TestNewLKEAPReranker_secretKeyFromExtraConfig(t *testing.T) {
|
||||
r, err := NewLKEAPReranker(&RerankerConfig{
|
||||
APIKey: "AKIDtest",
|
||||
ModelName: "lke-reranker-base",
|
||||
ExtraConfig: map[string]string{
|
||||
"secret_key": "sk-test",
|
||||
"region": "ap-beijing",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
assert.Equal(t, "lke-reranker-base", r.GetModelName())
|
||||
}
|
||||
|
||||
func TestNewLKEAPReranker_defaultModelName(t *testing.T) {
|
||||
r, err := NewLKEAPReranker(&RerankerConfig{
|
||||
APIKey: "AKIDtest",
|
||||
AppSecret: "sk-test",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, LKEAPDefaultRerankModel, r.GetModelName())
|
||||
}
|
||||
|
||||
func TestLKEAPReranker_Rerank_emptyDocuments(t *testing.T) {
|
||||
r, err := NewLKEAPReranker(&RerankerConfig{
|
||||
APIKey: "AKIDtest",
|
||||
AppSecret: "sk-test",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results, err := r.Rerank(t.Context(), "query", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, results)
|
||||
}
|
||||
|
||||
func TestLKEAPReranker_Rerank_batchesMoreThan60Documents(t *testing.T) {
|
||||
var batches [][]string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
var request struct {
|
||||
Docs []string `json:"Docs"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
batches = append(batches, request.Docs)
|
||||
|
||||
scores := make([]float64, len(request.Docs))
|
||||
for i, doc := range request.Docs {
|
||||
index, err := lkeapTestDocumentIndex(doc)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
scores[i] = float64(index)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Response": map[string]any{
|
||||
"ScoreList": scores,
|
||||
"RequestId": "test-request",
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
r := newTestLKEAPReranker(t, server.URL)
|
||||
documents := make([]string, 61)
|
||||
for i := range documents {
|
||||
documents[i] = strconv.Itoa(i) + ":document"
|
||||
}
|
||||
|
||||
results, err := r.Rerank(t.Context(), "query", documents)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, len(documents))
|
||||
require.Len(t, batches, 2)
|
||||
assert.Len(t, batches[0], 60)
|
||||
assert.Len(t, batches[1], 1)
|
||||
for i, result := range results {
|
||||
assert.Equal(t, i, result.Index)
|
||||
assert.Equal(t, documents[i], result.Document.Text)
|
||||
assert.Equal(t, float64(i), result.RelevanceScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLKEAPReranker_Rerank_batchesWithinCharacterLimit(t *testing.T) {
|
||||
var batches [][]string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
var request struct {
|
||||
Docs []string `json:"Docs"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
batches = append(batches, request.Docs)
|
||||
|
||||
scores := make([]float64, len(request.Docs))
|
||||
for i, doc := range request.Docs {
|
||||
index, err := lkeapTestDocumentIndex(doc)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
scores[i] = float64(index)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Response": map[string]any{
|
||||
"ScoreList": scores,
|
||||
"RequestId": "test-request",
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
r := newTestLKEAPReranker(t, server.URL)
|
||||
documents := []string{
|
||||
"0:" + strings.Repeat("a", 1000),
|
||||
"1:" + strings.Repeat("b", 1000),
|
||||
"2:" + strings.Repeat("c", 1000),
|
||||
}
|
||||
|
||||
results, err := r.Rerank(t.Context(), "query", documents)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, len(documents))
|
||||
require.Len(t, batches, 3)
|
||||
for i, batch := range batches {
|
||||
assert.Len(t, batch, 1)
|
||||
assert.Equal(t, documents[i], batch[0])
|
||||
}
|
||||
}
|
||||
|
||||
func newTestLKEAPReranker(t *testing.T, serverURL string) *LKEAPReranker {
|
||||
t.Helper()
|
||||
endpoint, err := url.Parse(serverURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
clientProfile := profile.NewClientProfile()
|
||||
clientProfile.HttpProfile.Endpoint = endpoint.Host
|
||||
clientProfile.HttpProfile.Scheme = "HTTP"
|
||||
client, err := lkeap.NewClient(common.NewCredential("AKIDtest", "sk-test"), LKEAPDefaultRegion, clientProfile)
|
||||
require.NoError(t, err)
|
||||
|
||||
return &LKEAPReranker{
|
||||
modelName: LKEAPDefaultRerankModel,
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
func lkeapTestDocumentIndex(document string) (int, error) {
|
||||
value, _, _ := strings.Cut(document, ":")
|
||||
return strconv.Atoi(value)
|
||||
}
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// NvidiaReranker implements a reranking system using Jina AI API
|
||||
// Jina API uses different parameters than standard OpenAI-compatible APIs
|
||||
type NvidiaReranker struct {
|
||||
modelName string // Name of the model used for reranking
|
||||
modelID string // Unique identifier of the model
|
||||
apiKey string // API key for authentication
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
customHeaders map[string]string
|
||||
}
|
||||
|
||||
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
func (r *NvidiaReranker) SetCustomHeaders(headers map[string]string) {
|
||||
r.customHeaders = headers
|
||||
}
|
||||
|
||||
type NvidiaRerankDocument struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// NvidiaRerankRequest represents a Jina rerank request
|
||||
// Note: Jina does NOT support truncate_prompt_tokens parameter
|
||||
type NvidiaRerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query NvidiaRerankDocument `json:"query"` // Query text to compare documents against
|
||||
Documents []NvidiaRerankDocument `json:"passages"` // List of document texts to rerank
|
||||
}
|
||||
|
||||
type NvidiaRankResult struct {
|
||||
Index int `json:"index"`
|
||||
Logit float64 `json:"logit"`
|
||||
}
|
||||
|
||||
// NvidiaRerankResponse represents the response from a Jina reranking request
|
||||
type NvidiaRerankResponse struct {
|
||||
Model string `json:"model"` // Model used for reranking
|
||||
Results []NvidiaRankResult `json:"rankings"` // Ranked results with relevance scores
|
||||
}
|
||||
|
||||
// NewNvidiaReranker creates a new instance of Jina reranker with the provided configuration
|
||||
func NewNvidiaReranker(config *RerankerConfig) (*NvidiaReranker, error) {
|
||||
apiKey := config.APIKey
|
||||
baseURL := "https://ai.api.nvidia.com/v1/retrieval/nvidia/reranking"
|
||||
if url := config.BaseURL; url != "" {
|
||||
baseURL = url
|
||||
}
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &NvidiaReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query
|
||||
func (r *NvidiaReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body - Jina does NOT use truncate_prompt_tokens
|
||||
requestBody := &NvidiaRerankRequest{
|
||||
Model: r.modelName,
|
||||
Query: NvidiaRerankDocument{Text: query},
|
||||
Documents: make([]NvidiaRerankDocument, len(documents)),
|
||||
}
|
||||
for i := range requestBody.Documents {
|
||||
requestBody.Documents[i].Text = documents[i]
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
// Send the request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
secutils.ApplyCustomHeaders(req, r.customHeaders)
|
||||
|
||||
// Log the curl equivalent for debugging (API key masked for security)
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
|
||||
r.baseURL, string(jsonData),
|
||||
)
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.GetLogger(ctx).Errorf("JinaReranker API error: Http Status: %s, Body: %s", resp.Status, string(body))
|
||||
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
|
||||
}
|
||||
|
||||
var response NvidiaRerankResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
ret := make([]RankResult, len(response.Results))
|
||||
for i, result := range response.Results {
|
||||
ret[i] = RankResult{
|
||||
Index: result.Index,
|
||||
Document: DocumentInfo{Text: documents[result.Index]},
|
||||
RelevanceScore: normalizeNvidiaLogit(result.Logit),
|
||||
}
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// normalizeNvidiaLogit converts NVIDIA's raw reranker logit into a probability.
|
||||
func normalizeNvidiaLogit(logit float64) float64 {
|
||||
if logit >= 0 {
|
||||
return 1 / (1 + math.Exp(-logit))
|
||||
}
|
||||
expLogit := math.Exp(logit)
|
||||
return expLogit / (1 + expLogit)
|
||||
}
|
||||
|
||||
// GetModelName returns the name of the reranking model
|
||||
func (r *NvidiaReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the unique identifier of the reranking model
|
||||
func (r *NvidiaReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNvidiaRerankerNormalizesLogitsToProbabilities(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPost, r.Method)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"model": "nvidia-rerank-model",
|
||||
"rankings": [
|
||||
{"index": 0, "logit": 23.0},
|
||||
{"index": 1, "logit": 0.0},
|
||||
{"index": 2, "logit": -23.0}
|
||||
]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
reranker := &NvidiaReranker{
|
||||
modelName: "nvidia-rerank-model",
|
||||
apiKey: "nvapi-test",
|
||||
baseURL: server.URL,
|
||||
client: server.Client(),
|
||||
}
|
||||
|
||||
results, err := reranker.Rerank(t.Context(), "query", []string{"high", "neutral", "low"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 3)
|
||||
|
||||
logits := []float64{23, 0, -23}
|
||||
for i, logit := range logits {
|
||||
want := 1 / (1 + math.Exp(-logit))
|
||||
assert.InDelta(t, want, results[i].RelevanceScore, 1e-12)
|
||||
assert.Greater(t, results[i].RelevanceScore, 0.0)
|
||||
assert.Less(t, results[i].RelevanceScore, 1.0)
|
||||
}
|
||||
|
||||
assert.Greater(t, results[0].RelevanceScore, results[1].RelevanceScore)
|
||||
assert.Greater(t, results[1].RelevanceScore, results[2].RelevanceScore)
|
||||
}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// protocolReranker adapts a protocol client to the Reranker interface and
|
||||
// owns the two things every vendor needs and none of them should implement
|
||||
// itself: splitting a candidate set that exceeds the documented per-request
|
||||
// ceilings, and putting the returned scores on one scale.
|
||||
type protocolReranker struct {
|
||||
inner api.Reranker
|
||||
settings catalog.RerankSettings
|
||||
endpoint string
|
||||
modelName string
|
||||
modelID string
|
||||
}
|
||||
|
||||
// defaultBatchConcurrency bounds in-flight batches when a vendor declares no
|
||||
// preference, so a large embedding_top_k cannot fan out into an unbounded
|
||||
// burst of requests.
|
||||
const defaultBatchConcurrency = 4
|
||||
|
||||
func (r *protocolReranker) GetModelName() string { return r.modelName }
|
||||
func (r *protocolReranker) GetModelID() string { return r.modelID }
|
||||
|
||||
func (r *protocolReranker) Rerank(
|
||||
ctx context.Context, query string, documents []string,
|
||||
) ([]RankResult, error) {
|
||||
if len(documents) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.endpoint, query, documents))
|
||||
|
||||
// The query travels in every request, so an over-long one cannot be made
|
||||
// to fit by splitting the documents. SplitBatches charges it to each
|
||||
// batch but only against a whole-request budget; a vendor that caps the
|
||||
// query on its own is checked here.
|
||||
if limit := r.settings.MaxQueryChars; limit > 0 {
|
||||
if length := utf8.RuneCountInString(query); length > limit {
|
||||
return nil, fmt.Errorf(
|
||||
"%s rerank: query is %d characters; the limit is %d",
|
||||
r.modelName, length, limit,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
batches, err := api.SplitBatches(documents, utf8.RuneCountInString(query), r.settings.BatchLimits())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s rerank: %w", r.modelName, err)
|
||||
}
|
||||
|
||||
// Each batch is scored against the same query independently, so the
|
||||
// per-batch results are comparable and are merged and re-ranked below.
|
||||
scored := make([][]api.RerankResult, len(batches))
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(r.concurrency())
|
||||
for i, batch := range batches {
|
||||
group.Go(func() error {
|
||||
out, err := r.inner.Rerank(groupCtx, query, batch.Items)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for j := range out {
|
||||
out[j].Index += batch.Start
|
||||
}
|
||||
scored[i] = out
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results := make([]RankResult, 0, len(documents))
|
||||
for _, batch := range scored {
|
||||
for _, item := range batch {
|
||||
if item.Index < 0 || item.Index >= len(documents) {
|
||||
return nil, fmt.Errorf(
|
||||
"%s rerank: index %d out of range for %d documents",
|
||||
r.modelName, item.Index, len(documents),
|
||||
)
|
||||
}
|
||||
text := item.Text
|
||||
if text == "" {
|
||||
text = documents[item.Index]
|
||||
}
|
||||
results = append(results, RankResult{
|
||||
Index: item.Index,
|
||||
Document: DocumentInfo{Text: text},
|
||||
RelevanceScore: normalizeScore(item.Score, r.settings.ScoreScale),
|
||||
})
|
||||
}
|
||||
}
|
||||
// A single request comes back ranked, and the retrieval pipeline reads
|
||||
// the slice as ranked: it takes results[0] as the best candidate for its
|
||||
// threshold fallback. Concatenating per-batch results would make that the
|
||||
// best of the first batch only, so the merged set is ordered once here.
|
||||
// Ties keep the lower index, which keeps the order stable.
|
||||
sort.SliceStable(results, func(i, j int) bool {
|
||||
return results[i].RelevanceScore > results[j].RelevanceScore
|
||||
})
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (r *protocolReranker) concurrency() int {
|
||||
if r.settings.MaxConcurrency > 0 {
|
||||
return r.settings.MaxConcurrency
|
||||
}
|
||||
return defaultBatchConcurrency
|
||||
}
|
||||
|
||||
// normalizeScore puts every vendor's score on the 0..1 scale the retrieval
|
||||
// pipeline compares against RerankThreshold.
|
||||
//
|
||||
// Most protocols already return a relevance probability. NIM returns the raw
|
||||
// logit of its relevance head instead — unbounded and routinely negative —
|
||||
// so a threshold tuned for probabilities would reject almost everything on
|
||||
// that vendor. The logistic function is the inverse of a log-odds, so this is
|
||||
// a unit conversion rather than a heuristic: it maps the vendor's own ranking
|
||||
// onto the scale the rest of the system already speaks, order preserved.
|
||||
func normalizeScore(score float64, scale api.ScoreScale) float64 {
|
||||
if scale != api.ScoreLogit {
|
||||
return score
|
||||
}
|
||||
// Two branches so neither exponential overflows: this is the form the
|
||||
// pre-catalog NVIDIA client used, kept verbatim.
|
||||
if score >= 0 {
|
||||
return 1 / (1 + math.Exp(-score))
|
||||
}
|
||||
exp := math.Exp(score)
|
||||
return exp / (1 + exp)
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeProtocol records every batch it was handed and scores documents by
|
||||
// their position within the batch, so a wrong offset shows up immediately.
|
||||
type fakeProtocol struct {
|
||||
mu sync.Mutex
|
||||
batches [][]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeProtocol) Rerank(
|
||||
_ context.Context, _ string, documents []string,
|
||||
) ([]api.RerankResult, error) {
|
||||
f.mu.Lock()
|
||||
f.batches = append(f.batches, append([]string(nil), documents...))
|
||||
f.mu.Unlock()
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
out := make([]api.RerankResult, len(documents))
|
||||
for i, doc := range documents {
|
||||
out[i] = api.RerankResult{Index: i, Score: float64(i), Text: doc}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func newWrapped(inner api.Reranker, settings catalog.RerankSettings) *protocolReranker {
|
||||
return &protocolReranker{
|
||||
inner: inner, settings: settings,
|
||||
endpoint: "https://example.invalid", modelName: "m", modelID: "id",
|
||||
}
|
||||
}
|
||||
|
||||
// The documented per-request ceilings live on the vendor, and one shared
|
||||
// layer enforces them. Indices must come back pointing into the caller's
|
||||
// original slice, not into the batch.
|
||||
func TestBatchingSplitsAndRestoresGlobalIndices(t *testing.T) {
|
||||
fake := &fakeProtocol{}
|
||||
r := newWrapped(fake, catalog.RerankSettings{MaxDocuments: 2, MaxConcurrency: 1})
|
||||
|
||||
docs := []string{"d0", "d1", "d2", "d3", "d4"}
|
||||
got, err := r.Rerank(context.Background(), "q", docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, fake.batches, 3, "5 documents at 2 per request is 3 requests")
|
||||
indices := make([]int, 0, len(got))
|
||||
for _, item := range got {
|
||||
indices = append(indices, item.Index)
|
||||
assert.Equal(t, docs[item.Index], item.Document.Text, "index must address the caller's slice")
|
||||
}
|
||||
assert.ElementsMatch(t, []int{0, 1, 2, 3, 4}, indices)
|
||||
}
|
||||
|
||||
func TestBatchingIsSkippedWhenTheVendorDeclaresNoLimit(t *testing.T) {
|
||||
fake := &fakeProtocol{}
|
||||
r := newWrapped(fake, catalog.RerankSettings{})
|
||||
|
||||
_, err := r.Rerank(context.Background(), "q", []string{"a", "b", "c", "d"})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, fake.batches, 1)
|
||||
}
|
||||
|
||||
// A document over the vendor's per-document ceiling cannot be scored. Saying
|
||||
// so beats sending a prefix and returning a number for text nobody asked
|
||||
// about.
|
||||
func TestAnOversizedDocumentIsAnError(t *testing.T) {
|
||||
r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxDocumentChars: 10})
|
||||
|
||||
_, err := r.Rerank(context.Background(), "q", []string{"ok", strings.Repeat("x", 99)})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "index 1")
|
||||
}
|
||||
|
||||
func TestBatchErrorsPropagate(t *testing.T) {
|
||||
r := newWrapped(&fakeProtocol{err: fmt.Errorf("upstream exploded")}, catalog.RerankSettings{})
|
||||
|
||||
_, err := r.Rerank(context.Background(), "q", []string{"a"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "upstream exploded")
|
||||
}
|
||||
|
||||
func TestEmptyInputMakesNoRequest(t *testing.T) {
|
||||
fake := &fakeProtocol{}
|
||||
r := newWrapped(fake, catalog.RerankSettings{})
|
||||
|
||||
got, err := r.Rerank(context.Background(), "q", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
assert.Empty(t, fake.batches)
|
||||
}
|
||||
|
||||
// NIM returns the raw logit of its relevance head. The retrieval pipeline
|
||||
// compares scores against RerankThreshold, which is tuned for probabilities,
|
||||
// so the logit is converted rather than passed through — otherwise the
|
||||
// default threshold of 0.2 keeps only documents whose logit is positive.
|
||||
func TestLogitScoresBecomeProbabilities(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
logit float64
|
||||
want float64
|
||||
}{
|
||||
{logit: 0.226318359375, want: 0.5563},
|
||||
{logit: -1.171875, want: 0.2366},
|
||||
{logit: -6.3125, want: 0.0018},
|
||||
{logit: 0, want: 0.5},
|
||||
} {
|
||||
assert.InDelta(t, tc.want, normalizeScore(tc.logit, api.ScoreLogit), 1e-4,
|
||||
"logit %v", tc.logit)
|
||||
}
|
||||
}
|
||||
|
||||
// The conversion is monotonic, so it re-scales without reordering.
|
||||
func TestLogitConversionPreservesOrder(t *testing.T) {
|
||||
logits := []float64{-6.3125, -1.171875, 0.226318359375, 4}
|
||||
previous := -1.0
|
||||
for _, logit := range logits {
|
||||
got := normalizeScore(logit, api.ScoreLogit)
|
||||
assert.Greater(t, got, previous)
|
||||
previous = got
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbabilityScoresPassThroughUntouched(t *testing.T) {
|
||||
assert.Equal(t, 0.9819, normalizeScore(0.9819, api.ScoreProbability))
|
||||
assert.Equal(t, 0.9819, normalizeScore(0.9819, ""))
|
||||
}
|
||||
|
||||
// A vendor that caps the query cannot be satisfied by splitting documents,
|
||||
// so the query is checked on its own before any request goes out.
|
||||
func TestAnOversizedQueryIsAnError(t *testing.T) {
|
||||
fake := &fakeProtocol{}
|
||||
r := newWrapped(fake, catalog.RerankSettings{MaxQueryChars: 10})
|
||||
|
||||
_, err := r.Rerank(context.Background(), strings.Repeat("q", 11), []string{"d"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "query is 11 characters")
|
||||
assert.Empty(t, fake.batches, "nothing should reach the vendor")
|
||||
}
|
||||
|
||||
func TestAQueryWithinTheLimitPasses(t *testing.T) {
|
||||
r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxQueryChars: 10})
|
||||
_, err := r.Rerank(context.Background(), strings.Repeat("中", 10), []string{"d"})
|
||||
require.NoError(t, err, "runes, not bytes")
|
||||
}
|
||||
|
||||
// The retrieval pipeline reads results[0] as the best candidate. Batches come
|
||||
// back each individually ranked, so the merged set has to be re-ranked or the
|
||||
// caller would see the best of the first batch only.
|
||||
func TestMergedBatchesAreRankedGlobally(t *testing.T) {
|
||||
// Scores are the position within the batch, so batch 2 holds the highest.
|
||||
r := newWrapped(&fakeProtocol{}, catalog.RerankSettings{MaxDocuments: 2, MaxConcurrency: 1})
|
||||
|
||||
got, err := r.Rerank(context.Background(), "q", []string{"d0", "d1", "d2", "d3", "d4"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 5)
|
||||
|
||||
for i := 1; i < len(got); i++ {
|
||||
assert.GreaterOrEqual(t, got[i-1].RelevanceScore, got[i].RelevanceScore,
|
||||
"results must be ranked across batches, not concatenated")
|
||||
}
|
||||
}
|
||||
|
||||
// The pre-catalog NVIDIA client used a two-branch logistic so neither
|
||||
// exponential overflows. These are the values its own test pinned.
|
||||
func TestLogitConversionHandlesExtremes(t *testing.T) {
|
||||
for _, logit := range []float64{23, 0, -23} {
|
||||
got := normalizeScore(logit, api.ScoreLogit)
|
||||
assert.InDelta(t, 1/(1+math.Exp(-logit)), got, 1e-12, "logit %v", logit)
|
||||
assert.Greater(t, got, 0.0)
|
||||
assert.Less(t, got, 1.0)
|
||||
}
|
||||
assert.False(t, math.IsNaN(normalizeScore(-1000, api.ScoreLogit)))
|
||||
assert.False(t, math.IsNaN(normalizeScore(1000, api.ScoreLogit)))
|
||||
}
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// OpenAIReranker implements a reranking system based on OpenAI models
|
||||
type OpenAIReranker struct {
|
||||
modelName string // Name of the model used for reranking
|
||||
modelID string // Unique identifier of the model
|
||||
apiKey string // API key for authentication
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
customHeaders map[string]string
|
||||
// truncatePromptTokens, when > 0, is sent as the vLLM-specific
|
||||
// truncate_prompt_tokens request field. It must never be sent by default:
|
||||
// providers that honor it (e.g. SiliconFlow) keep only the LAST N tokens of
|
||||
// the templated rerank prompt, which cuts the query off long documents and
|
||||
// collapses every relevance score to near zero (issue #2143).
|
||||
truncatePromptTokens int
|
||||
}
|
||||
|
||||
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
func (r *OpenAIReranker) SetCustomHeaders(headers map[string]string) {
|
||||
r.customHeaders = headers
|
||||
}
|
||||
|
||||
// RerankRequest represents a request to rerank documents based on relevance to a query
|
||||
type RerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query string `json:"query"` // Query text to compare documents against
|
||||
Documents []string `json:"documents"` // List of document texts to rerank
|
||||
AdditionalData map[string]interface{} `json:"additional_data,omitempty"` // Optional additional data for the model
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` // Maximum prompt tokens to use (vLLM-specific, opt-in)
|
||||
}
|
||||
|
||||
// RerankResponse represents the response from a reranking request
|
||||
type RerankResponse struct {
|
||||
ID string `json:"id"` // Request ID
|
||||
Model string `json:"model"` // Model used for reranking
|
||||
Usage UsageInfo `json:"usage"` // Token usage information
|
||||
Results []RankResult `json:"results"` // Ranked results with relevance scores
|
||||
}
|
||||
|
||||
// UsageInfo contains information about token usage in the API request
|
||||
type UsageInfo struct {
|
||||
TotalTokens int `json:"total_tokens"` // Total tokens consumed
|
||||
}
|
||||
|
||||
// NewOpenAIReranker creates a new instance of OpenAI reranker with the provided configuration
|
||||
func NewOpenAIReranker(config *RerankerConfig) (*OpenAIReranker, error) {
|
||||
apiKey := config.APIKey
|
||||
baseURL := "https://api.openai.com/v1"
|
||||
if url := config.BaseURL; url != "" {
|
||||
baseURL = url
|
||||
}
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Optional opt-in for vLLM-style deployments that need server-side prompt
|
||||
// truncation. Configured via extra_config; never enabled by default.
|
||||
truncatePromptTokens := 0
|
||||
if config.ExtraConfig != nil {
|
||||
if raw := strings.TrimSpace(config.ExtraConfig["truncate_prompt_tokens"]); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("invalid truncate_prompt_tokens in extra_config: %q", raw)
|
||||
}
|
||||
truncatePromptTokens = n
|
||||
}
|
||||
}
|
||||
|
||||
return &OpenAIReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
truncatePromptTokens: truncatePromptTokens,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query
|
||||
func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body. truncate_prompt_tokens is only included when
|
||||
// explicitly configured: sending it unconditionally corrupts scores on
|
||||
// providers that honor it (see OpenAIReranker.truncatePromptTokens).
|
||||
requestBody := &RerankRequest{
|
||||
Model: r.modelName,
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
TruncatePromptTokens: r.truncatePromptTokens,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
// Send the request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
secutils.ApplyCustomHeaders(req, r.customHeaders)
|
||||
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, fmt.Sprintf("%s/rerank", r.baseURL), query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status)
|
||||
}
|
||||
|
||||
var response RerankResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
return response.Results, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the name of the reranking model
|
||||
func (r *OpenAIReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the unique identifier of the reranking model
|
||||
func (r *OpenAIReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newRerankScoreTestServer emulates an OpenAI-compatible /rerank endpoint the
|
||||
// way a vLLM-backed provider (e.g. SiliconFlow) behaves, and records the last
|
||||
// decoded request body.
|
||||
//
|
||||
// If the request carries truncate_prompt_tokens, the backend keeps only the
|
||||
// last N tokens of the templated rerank prompt — the query gets cut off long
|
||||
// documents and every relevance score collapses to near zero (issue #2143).
|
||||
// Otherwise it returns the real scores, sorted by relevance_score descending,
|
||||
// with index pointing back at the original input-document position.
|
||||
func newRerankScoreTestServer(t *testing.T, lastRequest *map[string]interface{}) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req map[string]interface{}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Errorf("decode rerank request: %v", err)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
*lastRequest = req
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if _, truncated := req["truncate_prompt_tokens"]; truncated {
|
||||
// Query truncated away from the prompt: scores collapse.
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id": "rerank-collapsed",
|
||||
"results": [
|
||||
{"index": 0, "relevance_score": 0.00670681, "document": {"text": "A"}},
|
||||
{"index": 1, "relevance_score": 0.00587342, "document": {"text": "B"}},
|
||||
{"index": 2, "relevance_score": 0.00412907, "document": {"text": "C"}}
|
||||
],
|
||||
"usage": {"total_tokens": 42}
|
||||
}`))
|
||||
return
|
||||
}
|
||||
// Healthy response: sorted by relevance_score descending, index values
|
||||
// out of order relative to the input documents [A, B, C].
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id": "rerank-ok",
|
||||
"results": [
|
||||
{"index": 2, "relevance_score": 0.998, "document": {"text": "C"}},
|
||||
{"index": 0, "relevance_score": 0.51, "document": {"text": "A"}},
|
||||
{"index": 1, "relevance_score": 0.006, "document": {"text": "B"}}
|
||||
],
|
||||
"usage": {"total_tokens": 42}
|
||||
}`))
|
||||
}))
|
||||
}
|
||||
|
||||
// TestOpenAIRerankerDoesNotTruncatePromptByDefault is the regression test for
|
||||
// issue #2143: the generic OpenAI-compatible reranker must not send the
|
||||
// vLLM-specific truncate_prompt_tokens field unless explicitly configured, and
|
||||
// each input document must keep its own relevance score via the index field.
|
||||
func TestOpenAIRerankerDoesNotTruncatePromptByDefault(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
|
||||
var lastRequest map[string]interface{}
|
||||
server := newRerankScoreTestServer(t, &lastRequest)
|
||||
defer server.Close()
|
||||
|
||||
reranker, err := NewOpenAIReranker(&RerankerConfig{
|
||||
BaseURL: server.URL,
|
||||
ModelName: "Qwen/Qwen3-VL-Reranker-8B",
|
||||
APIKey: "sk-test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAIReranker: %v", err)
|
||||
}
|
||||
|
||||
documents := []string{"A", "B", "C"}
|
||||
results, err := reranker.Rerank(t.Context(), "query", documents)
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank: %v", err)
|
||||
}
|
||||
|
||||
if v, ok := lastRequest["truncate_prompt_tokens"]; ok {
|
||||
t.Errorf("request contains truncate_prompt_tokens=%v; it must not be sent unless configured", v)
|
||||
}
|
||||
if v, ok := lastRequest["additional_data"]; ok {
|
||||
t.Errorf("request contains additional_data=%v; empty optional fields must be omitted", v)
|
||||
}
|
||||
|
||||
if len(results) != len(documents) {
|
||||
t.Fatalf("got %d results, want %d", len(results), len(documents))
|
||||
}
|
||||
|
||||
// Results stay in the order returned by the API (sorted by score desc).
|
||||
if results[0].Index != 2 || results[0].RelevanceScore != 0.998 {
|
||||
t.Errorf("top result = {index: %d, score: %v}, want {index: 2, score: 0.998}",
|
||||
results[0].Index, results[0].RelevanceScore)
|
||||
}
|
||||
|
||||
// Each input document keeps its own score, resolved through the index field.
|
||||
wantScoreByDoc := map[string]float64{"C": 0.998, "A": 0.51, "B": 0.006}
|
||||
for _, rr := range results {
|
||||
if rr.Index < 0 || rr.Index >= len(documents) {
|
||||
t.Fatalf("result index %d out of range for %d documents", rr.Index, len(documents))
|
||||
}
|
||||
doc := documents[rr.Index]
|
||||
if want := wantScoreByDoc[doc]; rr.RelevanceScore != want {
|
||||
t.Errorf("document %q got score %v, want %v", doc, rr.RelevanceScore, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIRerankerTruncatePromptTokensOptIn verifies that deployments which
|
||||
// really need server-side prompt truncation (self-hosted vLLM with small-
|
||||
// context rerankers) can still opt in via extra_config.
|
||||
func TestOpenAIRerankerTruncatePromptTokensOptIn(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
|
||||
var lastRequest map[string]interface{}
|
||||
server := newRerankScoreTestServer(t, &lastRequest)
|
||||
defer server.Close()
|
||||
|
||||
reranker, err := NewOpenAIReranker(&RerankerConfig{
|
||||
BaseURL: server.URL,
|
||||
ModelName: "bge-reranker-base",
|
||||
APIKey: "sk-test",
|
||||
ExtraConfig: map[string]string{"truncate_prompt_tokens": "511"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAIReranker: %v", err)
|
||||
}
|
||||
|
||||
if _, err := reranker.Rerank(t.Context(), "query", []string{"A", "B", "C"}); err != nil {
|
||||
t.Fatalf("Rerank: %v", err)
|
||||
}
|
||||
|
||||
v, ok := lastRequest["truncate_prompt_tokens"]
|
||||
if !ok {
|
||||
t.Fatal("request is missing truncate_prompt_tokens despite extra_config opt-in")
|
||||
}
|
||||
if n, _ := v.(float64); n != 511 {
|
||||
t.Errorf("truncate_prompt_tokens = %v, want 511", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOpenAIRerankerRejectsInvalidTruncatePromptTokens(t *testing.T) {
|
||||
for _, raw := range []string{"abc", "-1", "0"} {
|
||||
_, err := NewOpenAIReranker(&RerankerConfig{
|
||||
ModelName: "rerank-test",
|
||||
ExtraConfig: map[string]string{"truncate_prompt_tokens": raw},
|
||||
})
|
||||
if err == nil {
|
||||
t.Errorf("NewOpenAIReranker accepted invalid truncate_prompt_tokens %q", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4,14 +4,18 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
||||
// provider.DetectProvider answers from the vendor catalog, which is empty
|
||||
// until the vendor packages have run their init. Without this import every
|
||||
// URL detects as "generic" and newReranker below falls through to the
|
||||
// OpenAI-compatible reranker for LKEAP / Volcengine / Jina / … rows that
|
||||
// carry no explicit provider id.
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/api/cohererank"
|
||||
"github.com/Tencent/WeKnora/internal/models/api/dashscoperank"
|
||||
"github.com/Tencent/WeKnora/internal/models/api/nimrerank"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
// catalog.Resolve answers from the vendor catalog, which is empty until
|
||||
// the vendor packages have run their init. Without this import every row
|
||||
// resolves to the generic vendor: LKEAP and Volcengine would lose their
|
||||
// signed clients and Aliyun its native protocol.
|
||||
_ "github.com/Tencent/WeKnora/internal/models/vendors"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
|
@ -131,45 +135,99 @@ func NewReranker(config *RerankerConfig) (Reranker, error) {
|
|||
return wrapRerankerLangfuse(r, nil)
|
||||
}
|
||||
|
||||
// customHeaderSetter 表示支持注入自定义 HTTP header 的 reranker 实现。
|
||||
type customHeaderSetter interface {
|
||||
SetCustomHeaders(map[string]string)
|
||||
}
|
||||
|
||||
// newReranker resolves the catalog and returns the protocol client for the
|
||||
// configured model, wrapped in the shared batching and score-scaling layer.
|
||||
// It mirrors chat.NewRemoteChat: the vendor's facts decide the protocol, the
|
||||
// URL and the credential, and this function knows no vendor names.
|
||||
func newReranker(config *RerankerConfig) (Reranker, error) {
|
||||
// Use provider field if set, otherwise detect from URL using provider registry
|
||||
providerName := provider.ProviderName(config.Provider)
|
||||
if providerName == "" {
|
||||
providerName = provider.DetectProvider(config.BaseURL)
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("rerank config is nil")
|
||||
}
|
||||
resolved, err := catalog.Resolve(catalog.Ref{
|
||||
Provider: config.Provider,
|
||||
Model: config.ModelName,
|
||||
BaseURL: config.BaseURL,
|
||||
ModelType: types.ModelTypeRerank,
|
||||
Extra: config.ExtraConfig,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateRerankBaseURL(resolved.BaseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var (
|
||||
reranker Reranker
|
||||
err error
|
||||
)
|
||||
switch providerName {
|
||||
case provider.ProviderAliyun:
|
||||
reranker, err = NewAliyunReranker(config)
|
||||
case provider.ProviderZhipu:
|
||||
reranker, err = NewZhipuReranker(config)
|
||||
case provider.ProviderJina:
|
||||
reranker, err = NewJinaReranker(config)
|
||||
case provider.ProviderNvidia:
|
||||
reranker, err = NewNvidiaReranker(config)
|
||||
case provider.ProviderWeKnoraCloud:
|
||||
reranker, err = NewWeKnoraCloudReranker(config)
|
||||
case provider.ProviderLKEAP:
|
||||
reranker, err = NewLKEAPReranker(config)
|
||||
case provider.ProviderVolcengine:
|
||||
reranker, err = NewVolcengineReranker(config)
|
||||
vendor := resolved.Vendor
|
||||
creds := catalog.Credentials{APIKey: config.APIKey, AppID: config.AppID, AppSecret: config.AppSecret}
|
||||
if creds.APIKey == "" {
|
||||
creds.APIKey = vendor.DefaultAPIKey
|
||||
}
|
||||
// A signing vendor with no identity pair would otherwise send unsigned
|
||||
// requests and fail at the far end, which is a worse error than this one.
|
||||
if vendor.Auth == catalog.AuthSigned {
|
||||
if creds.AppID == "" {
|
||||
return nil, fmt.Errorf("%s rerank: AppID is required", vendor.Name)
|
||||
}
|
||||
if creds.AppSecret == "" {
|
||||
return nil, fmt.Errorf("%s rerank: AppSecret is required", vendor.Name)
|
||||
}
|
||||
}
|
||||
// Vendor headers first so a user header cannot silently replace a vendor
|
||||
// beta flag, matching chat.NewRemoteChat.
|
||||
headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders))
|
||||
for k, v := range vendor.Headers {
|
||||
headers[k] = v
|
||||
}
|
||||
for k, v := range config.CustomHeaders {
|
||||
headers[k] = v
|
||||
}
|
||||
endpoint := api.Endpoint{
|
||||
BaseURL: resolved.BaseURL,
|
||||
Model: resolved.RemoteModel,
|
||||
ModelID: config.ModelID,
|
||||
Auth: vendor.AuthFunc(vendor.API, creds),
|
||||
Headers: headers,
|
||||
// Most vendors have always run without a client deadline here and let
|
||||
// the caller's context govern; the ones that declare a timeout get it.
|
||||
Client: newRerankHTTPClient(
|
||||
time.Duration(resolved.Rerank.RequestTimeout) * time.Second,
|
||||
),
|
||||
}
|
||||
if vendor.Endpoint != nil {
|
||||
if url, query := vendor.Endpoint(catalog.EndpointRequest{
|
||||
BaseURL: resolved.BaseURL,
|
||||
Model: resolved.RemoteModel,
|
||||
ModelType: types.ModelTypeRerank,
|
||||
Extra: config.ExtraConfig,
|
||||
}); url != "" {
|
||||
endpoint.URL, endpoint.Query = url, query
|
||||
}
|
||||
}
|
||||
|
||||
var client api.Reranker
|
||||
switch resolved.RerankAPI {
|
||||
case api.RerankCohere:
|
||||
client = cohererank.New(cohererank.Config{Endpoint: endpoint, Settings: resolved.Rerank})
|
||||
case api.RerankDashScope:
|
||||
client = dashscoperank.New(dashscoperank.Config{Endpoint: endpoint, Settings: resolved.Rerank})
|
||||
case api.RerankNIM:
|
||||
client = nimrerank.New(nimrerank.Config{Endpoint: endpoint, Settings: resolved.Rerank})
|
||||
case api.RerankTencentLKEAP:
|
||||
client, err = newLKEAPClient(config, resolved)
|
||||
case api.RerankVolcengineKnowledge:
|
||||
client, err = newVolcengineClient(config, resolved)
|
||||
default:
|
||||
reranker, err = NewOpenAIReranker(config)
|
||||
return nil, fmt.Errorf("unsupported rerank api %q for provider %s", resolved.RerankAPI, vendor.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if setter, ok := reranker.(customHeaderSetter); ok {
|
||||
setter.SetCustomHeaders(config.CustomHeaders)
|
||||
}
|
||||
return reranker, nil
|
||||
|
||||
return &protocolReranker{
|
||||
inner: client,
|
||||
settings: resolved.Rerank,
|
||||
endpoint: resolved.BaseURL,
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,152 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The two SDK-backed vendors have no protocol package, so their outbound
|
||||
// shape is not covered by a golden test. These pin what the SDK actually puts
|
||||
// on the wire, which is what the per-client tests did before the migration.
|
||||
|
||||
const volcengineRerankPath = "/api/knowledge/service/rerank"
|
||||
|
||||
func TestVolcengineClientOutboundShape(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
|
||||
var request struct {
|
||||
Datas []struct {
|
||||
Query string `json:"query"`
|
||||
Content *string `json:"content"`
|
||||
} `json:"datas"`
|
||||
RerankModel *string `json:"rerank_model"`
|
||||
RerankInstruction *string `json:"rerank_instruction"`
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodPost, r.Method)
|
||||
assert.Equal(t, volcengineRerankPath, r.URL.Path)
|
||||
// The access key identifies the caller in the signature; the secret
|
||||
// signs it and must never travel.
|
||||
assert.Contains(t, r.Header.Get("Authorization"), "AKLT-test")
|
||||
assert.NotContains(t, r.Header.Get("Authorization"), "secret-test")
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.91,0.27]}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := newVolcengineClient(
|
||||
&RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"},
|
||||
&catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := client.Rerank(context.Background(), "保留对话数据吗", []string{"会保留", "不会保留"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, 0, got[0].Index)
|
||||
assert.Equal(t, "会保留", got[0].Text)
|
||||
assert.InDelta(t, 0.91, got[0].Score, 1e-4)
|
||||
assert.Equal(t, 1, got[1].Index)
|
||||
assert.InDelta(t, 0.27, got[1].Score, 1e-4)
|
||||
|
||||
require.NotNil(t, request.RerankModel)
|
||||
assert.Equal(t, "doubao-seed-rerank", *request.RerankModel)
|
||||
require.Len(t, request.Datas, 2)
|
||||
assert.Equal(t, "保留对话数据吗", request.Datas[0].Query)
|
||||
require.NotNil(t, request.Datas[0].Content)
|
||||
assert.Equal(t, "会保留", *request.Datas[0].Content)
|
||||
require.NotNil(t, request.RerankInstruction)
|
||||
assert.Contains(t, *request.RerankInstruction, "Document")
|
||||
}
|
||||
|
||||
func TestVolcengineClientSurfacesAnInBodyErrorCode(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"code":1000030,"message":"quota exceeded","data":{}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := newVolcengineClient(
|
||||
&RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"},
|
||||
&catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.Rerank(context.Background(), "q", []string{"d"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "quota exceeded")
|
||||
}
|
||||
|
||||
// A score list that does not line up with the documents cannot be mapped back
|
||||
// onto them, so it is an error rather than a partially filled result.
|
||||
func TestVolcengineClientRejectsAScoreCountMismatch(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"code":0,"data":{"scores":[0.5]}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := newVolcengineClient(
|
||||
&RerankerConfig{APIKey: "AKLT-test", AppSecret: "secret-test"},
|
||||
&catalog.Resolved{BaseURL: server.URL, RemoteModel: "doubao-seed-rerank"},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.Rerank(context.Background(), "q", []string{"d0", "d1"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "score count mismatch")
|
||||
}
|
||||
|
||||
func TestVolcengineClientRequiresAKAndSK(t *testing.T) {
|
||||
for _, cfg := range []*RerankerConfig{
|
||||
{AppSecret: "sk"},
|
||||
{APIKey: "ak"},
|
||||
} {
|
||||
_, err := newVolcengineClient(cfg, &catalog.Resolved{BaseURL: "https://example.invalid"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "access key and secret key are required")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLKEAPClientRequiresCredentials(t *testing.T) {
|
||||
for _, cfg := range []*RerankerConfig{
|
||||
{AppSecret: "sk"},
|
||||
{APIKey: "id"},
|
||||
} {
|
||||
_, err := newLKEAPClient(cfg, &catalog.Resolved{})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "secret_id and secret_key are required")
|
||||
}
|
||||
}
|
||||
|
||||
// The secret key has always been accepted from extra_config as well, because
|
||||
// the model editor stores it there for this vendor.
|
||||
func TestLKEAPClientTakesTheSecretKeyFromExtraConfig(t *testing.T) {
|
||||
client, err := newLKEAPClient(
|
||||
&RerankerConfig{APIKey: "AKIDxxx", ExtraConfig: map[string]string{"secret_key": "sk"}},
|
||||
&catalog.Resolved{RemoteModel: "lke-reranker-base"},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, client)
|
||||
}
|
||||
|
||||
func TestLKEAPClientFallsBackToTheDefaultModel(t *testing.T) {
|
||||
client, err := newLKEAPClient(
|
||||
&RerankerConfig{APIKey: "AKIDxxx", AppSecret: "sk"},
|
||||
&catalog.Resolved{},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
inner, ok := client.(*lkeapClient)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, LKEAPDefaultRerankModel, inner.modelName)
|
||||
assert.False(t, strings.Contains(inner.modelName, " "))
|
||||
}
|
||||
|
|
@ -16,19 +16,19 @@ func withRerankSSRFWhitelist(t *testing.T, raw string) {
|
|||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
}
|
||||
|
||||
func TestOpenAIRerankerRejectsInternalBaseURL(t *testing.T) {
|
||||
func TestRerankerRejectsInternalBaseURL(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "")
|
||||
|
||||
_, err := NewOpenAIReranker(&RerankerConfig{
|
||||
_, err := NewReranker(&RerankerConfig{
|
||||
BaseURL: "http://169.254.169.254/latest/meta-data/",
|
||||
ModelName: "rerank-test",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("NewOpenAIReranker returned nil error for blocked internal BaseURL")
|
||||
t.Fatalf("NewReranker returned nil error for blocked internal BaseURL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIRerankerBlocksRedirectToInternalURL(t *testing.T) {
|
||||
func TestRerankerBlocksRedirectToInternalURL(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -36,13 +36,13 @@ func TestOpenAIRerankerBlocksRedirectToInternalURL(t *testing.T) {
|
|||
}))
|
||||
defer server.Close()
|
||||
|
||||
reranker, err := NewOpenAIReranker(&RerankerConfig{
|
||||
reranker, err := NewReranker(&RerankerConfig{
|
||||
BaseURL: server.URL,
|
||||
ModelName: "rerank-test",
|
||||
APIKey: "sk-test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAIReranker: %v", err)
|
||||
t.Fatalf("NewReranker: %v", err)
|
||||
}
|
||||
|
||||
_, err = reranker.Rerank(t.Context(), "query", []string{"doc"})
|
||||
|
|
|
|||
|
|
@ -6,37 +6,33 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/api"
|
||||
"github.com/Tencent/WeKnora/internal/models/catalog"
|
||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
||||
"github.com/volcengine/vikingdb-go-sdk/knowledge"
|
||||
knowledgemodel "github.com/volcengine/vikingdb-go-sdk/knowledge/model"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
// VolcengineRerankBaseURL is the managed Knowledge Service host.
|
||||
VolcengineRerankBaseURL = provider.VolcengineRerankBaseURL
|
||||
|
||||
volcengineRerankPath = "/api/knowledge/service/rerank"
|
||||
volcengineRerankDefaultModel = "doubao-seed-rerank"
|
||||
volcengineRerankDefaultRegion = "cn-beijing"
|
||||
volcengineRerankDefaultInstruction = "Whether the Document answers the Query or matches the content retrieval intent"
|
||||
volcengineRerankMaxDocuments = 50
|
||||
// volcengineRerankMaxConcurrency bounds the number of in-flight batch
|
||||
// requests when the candidate set exceeds volcengineRerankMaxDocuments, so a
|
||||
// very large embedding_top_k cannot fan out into an unbounded burst of calls.
|
||||
volcengineRerankMaxConcurrency = 4
|
||||
)
|
||||
|
||||
// VolcengineReranker calls the managed Knowledge Service Rerank API with AK/SK signing.
|
||||
type VolcengineReranker struct {
|
||||
// volcengineClient calls the managed Knowledge Service rerank through the
|
||||
// vikingdb SDK, which owns the AK/SK signing. It implements api.Reranker and
|
||||
// nothing else: the 50-document ceiling and the batch concurrency are
|
||||
// declared on the vendor and enforced by protocolReranker.
|
||||
type volcengineClient struct {
|
||||
modelName string
|
||||
instruction string
|
||||
modelID string
|
||||
endpoint string
|
||||
client *knowledge.Client
|
||||
}
|
||||
|
||||
func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error) {
|
||||
func newVolcengineClient(config *RerankerConfig, resolved *catalog.Resolved) (api.Reranker, error) {
|
||||
accessKey := strings.TrimSpace(config.APIKey)
|
||||
secretKey := strings.TrimSpace(config.AppSecret)
|
||||
if secretKey == "" && config.ExtraConfig != nil {
|
||||
|
|
@ -46,15 +42,12 @@ func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error)
|
|||
return nil, fmt.Errorf("access key and secret key are required for Volcengine rerank")
|
||||
}
|
||||
|
||||
baseURL := strings.TrimRight(strings.TrimSpace(config.BaseURL), "/")
|
||||
baseURL := resolved.BaseURL
|
||||
if baseURL == "" {
|
||||
baseURL = VolcengineRerankBaseURL
|
||||
}
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modelName := strings.TrimSpace(config.ModelName)
|
||||
modelName := strings.TrimSpace(resolved.RemoteModel)
|
||||
if modelName == "" {
|
||||
modelName = volcengineRerankDefaultModel
|
||||
}
|
||||
|
|
@ -80,70 +73,17 @@ func NewVolcengineReranker(config *RerankerConfig) (*VolcengineReranker, error)
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("create Volcengine rerank client: %w", err)
|
||||
}
|
||||
|
||||
return &VolcengineReranker{
|
||||
modelName: modelName,
|
||||
instruction: instruction,
|
||||
modelID: config.ModelID,
|
||||
endpoint: baseURL,
|
||||
client: client,
|
||||
}, nil
|
||||
return &volcengineClient{modelName: modelName, instruction: instruction, client: client}, nil
|
||||
}
|
||||
|
||||
func (r *VolcengineReranker) Rerank(
|
||||
// Rerank scores one batch. Every document is paired with the same query and
|
||||
// instruction, and the reply is a score list in input order.
|
||||
func (r *volcengineClient) Rerank(
|
||||
ctx context.Context, query string, documents []string,
|
||||
) ([]RankResult, error) {
|
||||
if len(documents) == 0 {
|
||||
return []RankResult{}, nil
|
||||
}
|
||||
|
||||
// The managed Knowledge Service Rerank API rejects requests carrying more
|
||||
// than volcengineRerankMaxDocuments items. Upstream callers (chat pipeline,
|
||||
// agent knowledge search, message search) feed in every retrieval candidate
|
||||
// and do not cap the count per provider, so a large embedding_top_k or a
|
||||
// multi-target search can exceed the limit. Each Data item is scored
|
||||
// independently against the same (query, instruction) pair, so the scores
|
||||
// are comparable across requests — we can split the documents into limit-
|
||||
// sized batches, rerank them concurrently, and merge without losing any
|
||||
// candidate (unlike truncation) or biasing the ranking.
|
||||
results := make([]RankResult, len(documents))
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(volcengineRerankMaxConcurrency)
|
||||
for start := 0; start < len(documents); start += volcengineRerankMaxDocuments {
|
||||
start := start
|
||||
end := min(start+volcengineRerankMaxDocuments, len(documents))
|
||||
g.Go(func() error {
|
||||
scores, err := r.rerankBatch(gctx, query, documents[start:end])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, score := range scores {
|
||||
results[start+i] = RankResult{
|
||||
Index: start + i,
|
||||
Document: DocumentInfo{Text: documents[start+i]},
|
||||
RelevanceScore: score,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// rerankBatch scores a single batch of documents (already sized within the API
|
||||
// limit) and returns the per-document relevance scores in input order.
|
||||
func (r *VolcengineReranker) rerankBatch(
|
||||
ctx context.Context, query string, documents []string,
|
||||
) ([]float64, error) {
|
||||
) ([]api.RerankResult, error) {
|
||||
data := make([]knowledgemodel.RerankDataItem, len(documents))
|
||||
for i := range documents {
|
||||
data[i] = knowledgemodel.RerankDataItem{
|
||||
Query: query,
|
||||
Content: &documents[i],
|
||||
}
|
||||
data[i] = knowledgemodel.RerankDataItem{Query: query, Content: &documents[i]}
|
||||
}
|
||||
request := knowledgemodel.RerankRequest{
|
||||
Datas: data,
|
||||
|
|
@ -151,11 +91,6 @@ func (r *VolcengineReranker) rerankBatch(
|
|||
RerankInstruction: &r.instruction,
|
||||
}
|
||||
|
||||
logger.Debugf(
|
||||
ctx,
|
||||
"%s",
|
||||
buildRerankRequestDebug(r.modelName, r.endpoint+volcengineRerankPath, query, documents),
|
||||
)
|
||||
response, err := r.client.Rerank(ctx, request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call Volcengine rerank: %w", err)
|
||||
|
|
@ -169,17 +104,13 @@ func (r *VolcengineReranker) rerankBatch(
|
|||
if len(response.Data.Scores) != len(documents) {
|
||||
return nil, fmt.Errorf(
|
||||
"Volcengine rerank score count mismatch: got %d scores for %d documents",
|
||||
len(response.Data.Scores),
|
||||
len(documents),
|
||||
len(response.Data.Scores), len(documents),
|
||||
)
|
||||
}
|
||||
return response.Data.Scores, nil
|
||||
}
|
||||
|
||||
func (r *VolcengineReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
func (r *VolcengineReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
results := make([]api.RerankResult, len(documents))
|
||||
for i, score := range response.Data.Scores {
|
||||
results[i] = api.RerankResult{Index: i, Score: score, Text: documents[i]}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,178 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVolcengineReranker_Rerank(t *testing.T) {
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
var request struct {
|
||||
Datas []struct {
|
||||
Query string `json:"query"`
|
||||
Content *string `json:"content"`
|
||||
} `json:"datas"`
|
||||
RerankModel *string `json:"rerank_model"`
|
||||
RerankInstruction *string `json:"rerank_instruction"`
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodPost, r.Method)
|
||||
assert.Equal(t, volcengineRerankPath, r.URL.Path)
|
||||
assert.Contains(t, r.Header.Get("Authorization"), "AKLT-test")
|
||||
assert.NotContains(t, r.Header.Get("Authorization"), "secret-test")
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.91,0.27]}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
reranker, err := NewVolcengineReranker(&RerankerConfig{
|
||||
APIKey: "AKLT-test",
|
||||
AppSecret: "secret-test",
|
||||
BaseURL: server.URL,
|
||||
ModelName: "doubao-seed-rerank",
|
||||
ModelID: "volc-rerank",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results, err := reranker.Rerank(t.Context(), "保留对话数据吗", []string{"会保留", "不会保留"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 2)
|
||||
assert.Equal(t, 0, results[0].Index)
|
||||
assert.Equal(t, "会保留", results[0].Document.Text)
|
||||
assert.InDelta(t, 0.91, results[0].RelevanceScore, 0.0001)
|
||||
assert.Equal(t, 1, results[1].Index)
|
||||
assert.Equal(t, "不会保留", results[1].Document.Text)
|
||||
assert.InDelta(t, 0.27, results[1].RelevanceScore, 0.0001)
|
||||
|
||||
require.NotNil(t, request.RerankModel)
|
||||
assert.Equal(t, "doubao-seed-rerank", *request.RerankModel)
|
||||
require.Len(t, request.Datas, 2)
|
||||
assert.Equal(t, "保留对话数据吗", request.Datas[0].Query)
|
||||
require.NotNil(t, request.Datas[0].Content)
|
||||
assert.Equal(t, "会保留", *request.Datas[0].Content)
|
||||
require.NotNil(t, request.RerankInstruction)
|
||||
assert.True(t, strings.Contains(*request.RerankInstruction, "Document"))
|
||||
}
|
||||
|
||||
func TestNewVolcengineReranker_RequiresAKSK(t *testing.T) {
|
||||
_, err := NewVolcengineReranker(&RerankerConfig{
|
||||
APIKey: "ark-api-key-only",
|
||||
BaseURL: VolcengineRerankBaseURL,
|
||||
ModelName: "doubao-seed-rerank",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "access key and secret key")
|
||||
}
|
||||
|
||||
func newTestVolcengineReranker(t *testing.T, handler http.HandlerFunc) *VolcengineReranker {
|
||||
t.Helper()
|
||||
withRerankSSRFWhitelist(t, "127.0.0.1")
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
reranker, err := NewVolcengineReranker(&RerankerConfig{
|
||||
APIKey: "AKLT-test",
|
||||
AppSecret: "secret-test",
|
||||
BaseURL: server.URL,
|
||||
ModelName: "doubao-seed-rerank",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return reranker
|
||||
}
|
||||
|
||||
func TestVolcengineReranker_EmptyDocuments(t *testing.T) {
|
||||
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Fatalf("rerank endpoint should not be called for empty documents")
|
||||
})
|
||||
|
||||
results, err := reranker.Rerank(t.Context(), "query", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, results)
|
||||
}
|
||||
|
||||
// TestVolcengineReranker_BatchesOverLimit verifies that a candidate set larger
|
||||
// than the API document limit is split into batches and reranked in full,
|
||||
// rather than truncated: every document gets a score and no batch exceeds the
|
||||
// per-request limit.
|
||||
func TestVolcengineReranker_BatchesOverLimit(t *testing.T) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
batchSizes []int
|
||||
requestCount int
|
||||
)
|
||||
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Datas []struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"datas"`
|
||||
}
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&body))
|
||||
|
||||
mu.Lock()
|
||||
requestCount++
|
||||
batchSizes = append(batchSizes, len(body.Datas))
|
||||
mu.Unlock()
|
||||
|
||||
scores := make([]string, len(body.Datas))
|
||||
for i := range scores {
|
||||
scores[i] = "0.5"
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[` +
|
||||
strings.Join(scores, ",") + `]}}`))
|
||||
})
|
||||
|
||||
total := volcengineRerankMaxDocuments*2 + 10
|
||||
documents := make([]string, total)
|
||||
for i := range documents {
|
||||
documents[i] = fmt.Sprintf("doc-%d", i)
|
||||
}
|
||||
|
||||
results, err := reranker.Rerank(t.Context(), "query", documents)
|
||||
require.NoError(t, err)
|
||||
|
||||
// All documents reranked, each mapped back to its original index/text.
|
||||
require.Len(t, results, total)
|
||||
for i := range results {
|
||||
assert.Equal(t, i, results[i].Index)
|
||||
assert.Equal(t, documents[i], results[i].Document.Text)
|
||||
}
|
||||
|
||||
// Split into ceil(total/limit) batches, none exceeding the API limit.
|
||||
assert.Equal(t, 3, requestCount)
|
||||
for _, size := range batchSizes {
|
||||
assert.LessOrEqual(t, size, volcengineRerankMaxDocuments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolcengineReranker_APIErrorCode(t *testing.T) {
|
||||
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":100004,"message":"quota exceeded","data":{}}`))
|
||||
})
|
||||
|
||||
_, err := reranker.Rerank(t.Context(), "query", []string{"a", "b"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "100004")
|
||||
assert.Contains(t, err.Error(), "quota exceeded")
|
||||
}
|
||||
|
||||
func TestVolcengineReranker_ScoreCountMismatch(t *testing.T) {
|
||||
reranker := newTestVolcengineReranker(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":0,"message":"success","data":{"scores":[0.9]}}`))
|
||||
})
|
||||
|
||||
_, err := reranker.Rerank(t.Context(), "query", []string{"a", "b"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "score count mismatch")
|
||||
}
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const weKnoraCloudRerankPath = "/api/v1/rerank"
|
||||
|
||||
// WeKnoraCloudReranker 实现 rerank.Reranker 接口,对接 WeKnoraCloud /api/v1/rerank
|
||||
type WeKnoraCloudReranker struct {
|
||||
modelName string
|
||||
remoteModelName string
|
||||
modelID string
|
||||
appID string
|
||||
apiKey string
|
||||
baseURL string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewWeKnoraCloudReranker 构造 WeKnoraCloudReranker
|
||||
func NewWeKnoraCloudReranker(config *RerankerConfig) (*WeKnoraCloudReranker, error) {
|
||||
if config.AppID == "" {
|
||||
return nil, fmt.Errorf("WeKnoraCloud reranker: AppID is required")
|
||||
}
|
||||
if config.AppSecret == "" {
|
||||
return nil, fmt.Errorf("WeKnoraCloud reranker: AppSecret is required")
|
||||
}
|
||||
baseURL := strings.TrimRight(config.BaseURL, "/")
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteModelName := ""
|
||||
if config.ExtraConfig != nil {
|
||||
remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"])
|
||||
}
|
||||
return &WeKnoraCloudReranker{
|
||||
modelName: config.ModelName,
|
||||
remoteModelName: remoteModelName,
|
||||
modelID: config.ModelID,
|
||||
appID: config.AppID,
|
||||
apiKey: config.AppSecret,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(60 * time.Second),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type weKnoraCloudRerankRequest struct {
|
||||
Model string `json:"model"`
|
||||
Query string `json:"query"`
|
||||
Documents []string `json:"documents"`
|
||||
}
|
||||
|
||||
type weKnoraCloudRerankResponse struct {
|
||||
Results []struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore float64 `json:"relevance_score"`
|
||||
Document struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"document"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
func (r *WeKnoraCloudReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
reqBody := weKnoraCloudRerankRequest{
|
||||
Model: r.effectiveModelName(),
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
}
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("weknoracloud reranker: marshal: %w", err)
|
||||
}
|
||||
|
||||
requestID := uuid.New().String()
|
||||
headers := utils.Sign(r.appID, r.apiKey, requestID, string(bodyBytes))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.baseURL+weKnoraCloudRerankPath, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("weknoracloud reranker: create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("weknoracloud reranker: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("weknoracloud reranker: read response: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("weknoracloud reranker: status %d: %s", resp.StatusCode, string(respBytes))
|
||||
}
|
||||
|
||||
var rerankResp weKnoraCloudRerankResponse
|
||||
if err := json.Unmarshal(respBytes, &rerankResp); err != nil {
|
||||
return nil, fmt.Errorf("weknoracloud reranker: unmarshal: %w", err)
|
||||
}
|
||||
|
||||
results := make([]RankResult, 0, len(rerankResp.Results))
|
||||
for _, item := range rerankResp.Results {
|
||||
results = append(results, RankResult{
|
||||
Index: item.Index,
|
||||
RelevanceScore: item.RelevanceScore,
|
||||
Document: DocumentInfo{Text: item.Document.Text},
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (r *WeKnoraCloudReranker) effectiveModelName() string {
|
||||
if r.remoteModelName != "" {
|
||||
return r.remoteModelName
|
||||
}
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
func (r *WeKnoraCloudReranker) GetModelName() string { return r.modelName }
|
||||
func (r *WeKnoraCloudReranker) GetModelID() string { return r.modelID }
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
package rerank
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// ZhipuReranker implements a reranking system based on Zhipu AI models
|
||||
type ZhipuReranker struct {
|
||||
modelName string // Name of the model used for reranking
|
||||
modelID string // Unique identifier of the model
|
||||
apiKey string // API key for authentication
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
customHeaders map[string]string
|
||||
}
|
||||
|
||||
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
func (r *ZhipuReranker) SetCustomHeaders(headers map[string]string) {
|
||||
r.customHeaders = headers
|
||||
}
|
||||
|
||||
// ZhipuRerankRequest represents a request to rerank documents using Zhipu AI API
|
||||
type ZhipuRerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query string `json:"query"` // Query text to compare documents against
|
||||
Documents []string `json:"documents"` // List of document texts to rerank
|
||||
TopN int `json:"top_n,omitempty"` // Number of top results to return (0 = all)
|
||||
ReturnDocuments bool `json:"return_documents,omitempty"` // Whether to return documents in response
|
||||
ReturnRawScores bool `json:"return_raw_scores,omitempty"` // Whether to return raw scores
|
||||
}
|
||||
|
||||
// ZhipuRerankResponse represents the response from Zhipu AI reranking request
|
||||
type ZhipuRerankResponse struct {
|
||||
RequestID string `json:"request_id"` // Request ID from client or platform
|
||||
ID string `json:"id"` // Task order ID from Zhipu platform
|
||||
Results []ZhipuRankResult `json:"results"` // Ranked results with relevance scores
|
||||
Usage ZhipuUsage `json:"usage"` // Token usage information
|
||||
}
|
||||
|
||||
// ZhipuRankResult represents a single reranking result from Zhipu AI
|
||||
type ZhipuRankResult struct {
|
||||
Index int `json:"index"` // Original index of the document
|
||||
RelevanceScore float64 `json:"relevance_score"` // Relevance score
|
||||
Document string `json:"document,omitempty"` // Document text (optional)
|
||||
}
|
||||
|
||||
// ZhipuUsage contains information about token usage in the Zhipu API request
|
||||
type ZhipuUsage struct {
|
||||
TotalTokens int `json:"total_tokens"` // Total tokens consumed
|
||||
PromptTokens int `json:"prompt_tokens"` // Prompt tokens
|
||||
}
|
||||
|
||||
// NewZhipuReranker creates a new instance of Zhipu reranker with the provided configuration
|
||||
func NewZhipuReranker(config *RerankerConfig) (*ZhipuReranker, error) {
|
||||
apiKey := config.APIKey
|
||||
baseURL := "https://open.bigmodel.cn/api/paas/v4/rerank"
|
||||
if url := config.BaseURL; url != "" {
|
||||
baseURL = url
|
||||
}
|
||||
if err := validateRerankBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ZhipuReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query using Zhipu AI API
|
||||
func (r *ZhipuReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body
|
||||
requestBody := &ZhipuRerankRequest{
|
||||
Model: r.modelName,
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
TopN: 0, // Return all documents
|
||||
ReturnDocuments: true,
|
||||
ReturnRawScores: false,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
|
||||
// Send the request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
|
||||
secutils.ApplyCustomHeaders(req, r.customHeaders)
|
||||
|
||||
logger.Debugf(ctx, "%s", buildRerankRequestDebug(r.modelName, r.baseURL, query, documents))
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("zhipu rerank API error: Http Status: %s, Body: %s", resp.Status, string(body))
|
||||
}
|
||||
|
||||
var response ZhipuRerankResponse
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
// Convert Zhipu results to standard RankResult format
|
||||
results := make([]RankResult, len(response.Results))
|
||||
for i, zhipuResult := range response.Results {
|
||||
results[i] = RankResult{
|
||||
Index: zhipuResult.Index,
|
||||
Document: DocumentInfo{
|
||||
Text: zhipuResult.Document,
|
||||
},
|
||||
RelevanceScore: zhipuResult.RelevanceScore,
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// GetModelName returns the name of the reranking model
|
||||
func (r *ZhipuReranker) GetModelName() string {
|
||||
return r.modelName
|
||||
}
|
||||
|
||||
// GetModelID returns the unique identifier of the reranking model
|
||||
func (r *ZhipuReranker) GetModelID() string {
|
||||
return r.modelID
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@
|
|||
{"id": "text-embedding-v4", "name": "Tongyi text-embedding-v4", "type": "Embedding", "dimension": 1024, "source": "https://help.aliyun.com/zh/model-studio/embedding"},
|
||||
{"id": "text-embedding-v3", "name": "Tongyi text-embedding-v3", "type": "Embedding", "dimension": 1024, "source": "https://help.aliyun.com/zh/model-studio/embedding"},
|
||||
{"id": "tongyi-embedding-vision-plus", "name": "Tongyi Embedding Vision Plus", "type": "Embedding", "dimension": 1152, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api"},
|
||||
{"id": "qwen3-rerank", "name": "Qwen3 Rerank", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
|
||||
{"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
|
||||
{"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,12 +58,16 @@
|
|||
// - rerank is a separate DashScope-native endpoint
|
||||
// (/api/v1/services/rerank/text-rerank/text-rerank).
|
||||
//
|
||||
// unverified: the docs place qwen3-rerank on a different path than every
|
||||
// other rerank model (/compatible-api/v1/reranks instead of the native
|
||||
// text-rerank path this package defaults to), and WeKnora's DashScope rerank
|
||||
// client only speaks the native request shape. The single RerankBaseURL is
|
||||
// kept; qwen3-rerank needs an operator-supplied base URL until the client
|
||||
// learns the second shape
|
||||
// qwen3-rerank is a second, incompatible rerank protocol on the same vendor.
|
||||
// The text-rerank page puts it on /compatible-api/v1/reranks and states
|
||||
// outright that "两种接口的请求体结构和响应格式不同": its request is flat
|
||||
// (query / documents at the top level, no input/parameters wrapper) and its
|
||||
// response carries `results` at the top level with no `output` object. This
|
||||
// package implements only the native shape that gte-rerank-v2 and
|
||||
// qwen3.7-text-rerank use, so the entry is marked deprecated: it stays
|
||||
// resolvable for a row that already names it, but the picker no longer offers
|
||||
// a model that would be sent to the wrong path and decoded with the wrong
|
||||
// shape. Serving it needs a fourth rerank protocol package
|
||||
// (https://help.aliyun.com/zh/model-studio/text-rerank-api).
|
||||
//
|
||||
// unverified: no page states whether `prompt_cache_key` is accepted, so the
|
||||
|
|
@ -108,7 +112,7 @@ func init() {
|
|||
ID: ID,
|
||||
Name: "Alibaba Cloud DashScope",
|
||||
Names: map[string]string{"zh-CN": "阿里云 DashScope"},
|
||||
Description: "qwen-plus, qwen3.8-max, deepseek-v4-pro, text-embedding-v4, qwen3-rerank, etc.",
|
||||
Description: "qwen-plus, qwen3.8-max, deepseek-v4-pro, text-embedding-v4, gte-rerank-v2, etc.",
|
||||
Website: "https://bailian.console.aliyun.com",
|
||||
Icon: icon,
|
||||
API: api.APIOpenAICompletions,
|
||||
|
|
@ -128,7 +132,16 @@ func init() {
|
|||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
},
|
||||
RerankAPI: api.RerankDashScope,
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
SendReturnDocs: catalog.Ptr(true),
|
||||
// 500 documents per request for the native text-rerank models.
|
||||
// The query (4,000 tokens) and per-document limits are stated in
|
||||
// tokens, which a rune count cannot express, so they are not
|
||||
// declared.
|
||||
MaxDocuments: catalog.Ptr(500),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
// Explicit although it matches the protocol default, because
|
||||
// this vendor's own parameter table deprecates the other
|
||||
|
|
|
|||
|
|
@ -148,6 +148,9 @@ func init() {
|
|||
// api-version. A value switches back to the dated
|
||||
// /openai/deployments/... path.
|
||||
Placeholder: "leave empty for /openai/v1, or e.g. 2025-04-01-preview",
|
||||
Placeholders: map[string]string{
|
||||
"zh-CN": "留空走 /openai/v1,或填如 2025-04-01-preview",
|
||||
},
|
||||
},
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
|
|
|
|||
|
|
@ -71,7 +71,49 @@ func init() {
|
|||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
ExtraFields: []catalog.ExtraField{{
|
||||
Key: catalog.ExtraScoreScale,
|
||||
Label: "Rerank score scale",
|
||||
Labels: map[string]string{"zh-CN": "Rerank 分数标度"},
|
||||
Type: "select",
|
||||
Options: []catalog.ExtraFieldOption{
|
||||
{
|
||||
Label: "Unbounded score (Qwen3-Reranker class)",
|
||||
Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
|
||||
Value: "logit",
|
||||
},
|
||||
{
|
||||
Label: "0..1 relevance (BGE class)",
|
||||
Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
|
||||
Value: "probability",
|
||||
},
|
||||
},
|
||||
Placeholder: "match the reranker actually deployed behind this endpoint",
|
||||
Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
|
||||
// A gateway serves whatever reranker was deployed behind it, and
|
||||
// the two families disagree. The vendor default follows the
|
||||
// documentation; the operator knows which model is actually there.
|
||||
Required: false,
|
||||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}, {
|
||||
Key: catalog.ExtraTruncatePromptTokens,
|
||||
Label: "Rerank prompt truncation (vLLM)",
|
||||
Labels: map[string]string{"zh-CN": "Rerank 提示词截断长度(vLLM)"},
|
||||
Type: "number",
|
||||
// Off unless the operator asks: a runtime that does not implement
|
||||
// the extension rejects the unknown field.
|
||||
Required: false,
|
||||
Placeholder: "empty unless the backend rejects long documents",
|
||||
Placeholders: map[string]string{"zh-CN": "留空,除非后端因文档过长而报错"},
|
||||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}},
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// Any OpenAI-compatible endpoint an operator points here is most
|
||||
// often a vLLM or SGLang server, which is where
|
||||
// truncate_prompt_tokens comes from.
|
||||
AcceptsTruncatePromptTokens: catalog.Ptr(true),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
MaxTokensField: catalog.Ptr("max_tokens"),
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs),
|
||||
|
|
|
|||
|
|
@ -81,7 +81,55 @@ func init() {
|
|||
types.ModelTypeVLLM,
|
||||
types.ModelTypeASR,
|
||||
},
|
||||
ExtraFields: []catalog.ExtraField{{
|
||||
Key: catalog.ExtraScoreScale,
|
||||
Label: "Rerank score scale",
|
||||
Labels: map[string]string{"zh-CN": "Rerank 分数标度"},
|
||||
Type: "select",
|
||||
Options: []catalog.ExtraFieldOption{
|
||||
{
|
||||
Label: "Unbounded score (Qwen3-Reranker class)",
|
||||
Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
|
||||
Value: "logit",
|
||||
},
|
||||
{
|
||||
Label: "0..1 relevance (BGE class)",
|
||||
Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
|
||||
Value: "probability",
|
||||
},
|
||||
},
|
||||
Placeholder: "match the reranker actually deployed behind this endpoint",
|
||||
Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
|
||||
// A gateway serves whatever reranker was deployed behind it, and
|
||||
// the two families disagree. The vendor default follows the
|
||||
// documentation; the operator knows which model is actually there.
|
||||
Required: false,
|
||||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}, {
|
||||
Key: catalog.ExtraTruncatePromptTokens,
|
||||
Label: "Rerank prompt truncation (vLLM)",
|
||||
Labels: map[string]string{"zh-CN": "Rerank 提示词截断长度(vLLM)"},
|
||||
Type: "number",
|
||||
// Off unless the operator asks: a runtime that does not implement
|
||||
// the extension rejects the unknown field.
|
||||
Required: false,
|
||||
Placeholder: "empty unless the backend rejects long documents",
|
||||
Placeholders: map[string]string{"zh-CN": "留空,除非后端因文档过长而报错"},
|
||||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}},
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// GPUStack's built-in backends are vLLM, SGLang, Ascend MindIE and
|
||||
// VoxBox, so the vLLM rerank extension reaches the model.
|
||||
AcceptsTruncatePromptTokens: catalog.Ptr(true),
|
||||
// The field is spelled relevance_score like the Cohere shape, but
|
||||
// it is not a probability: the rerank API page's own example
|
||||
// returns 1.951932668685913, -3.7347371578216553 and
|
||||
// -6.157620906829834. Read as a probability, every negative score
|
||||
// falls below a relevance threshold tuned for 0..1.
|
||||
// https://docs.gpustack.ai/2.0/user-guide/rerank-api/
|
||||
ScoreScale: catalog.Ptr(api.ScoreLogit),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
MaxTokensField: catalog.Ptr("max_tokens"),
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs),
|
||||
|
|
|
|||
|
|
@ -63,6 +63,14 @@ func init() {
|
|||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// return_documents echoes the text back. Results are matched
|
||||
// by index, so this is not needed to map them; it is kept
|
||||
// because it is what this vendor has always been sent.
|
||||
SendReturnDocs: catalog.Ptr(true),
|
||||
},
|
||||
},
|
||||
Models: catalog.MustParseModels(modelsJSON),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ const ID = "litellm"
|
|||
// BaseURL is a placeholder the operator replaces with a reachable proxy.
|
||||
const BaseURL = "http://your_litellm_proxy/v1"
|
||||
|
||||
// RerankBaseURL is the proxy root. LiteLLM documents rerank at /rerank
|
||||
// rather than /v1/rerank, so it does not hang off the chat base URL.
|
||||
const RerankBaseURL = "http://your_litellm_proxy"
|
||||
|
||||
func init() {
|
||||
catalog.Register(&catalog.Vendor{
|
||||
ID: ID,
|
||||
|
|
@ -65,13 +69,40 @@ func init() {
|
|||
DefaultBaseURLs: map[types.ModelType]string{
|
||||
types.ModelTypeKnowledgeQA: BaseURL,
|
||||
types.ModelTypeEmbedding: BaseURL,
|
||||
types.ModelTypeRerank: RerankBaseURL,
|
||||
types.ModelTypeVLLM: BaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
},
|
||||
ExtraFields: []catalog.ExtraField{{
|
||||
Key: catalog.ExtraScoreScale,
|
||||
Label: "Rerank score scale",
|
||||
Labels: map[string]string{"zh-CN": "Rerank 分数标度"},
|
||||
Type: "select",
|
||||
Options: []catalog.ExtraFieldOption{
|
||||
{
|
||||
Label: "Unbounded score (Qwen3-Reranker class)",
|
||||
Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
|
||||
Value: "logit",
|
||||
},
|
||||
{
|
||||
Label: "0..1 relevance (BGE class)",
|
||||
Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
|
||||
Value: "probability",
|
||||
},
|
||||
},
|
||||
Placeholder: "match the reranker actually deployed behind this endpoint",
|
||||
Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
|
||||
// A gateway serves whatever reranker was deployed behind it, and
|
||||
// the two families disagree. The vendor default follows the
|
||||
// documentation; the operator knows which model is actually there.
|
||||
Required: false,
|
||||
ModelTypes: []types.ModelType{types.ModelTypeRerank},
|
||||
}},
|
||||
Compat: catalog.VendorCompat{
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatOpenAI),
|
||||
|
|
|
|||
|
|
@ -111,8 +111,11 @@ func init() {
|
|||
Type: "password",
|
||||
Required: true,
|
||||
Placeholder: "Tencent Cloud SecretKey (the API Key field holds the SecretId)",
|
||||
ModelTypes: rerankOnly,
|
||||
Secret: true,
|
||||
Placeholders: map[string]string{
|
||||
"zh-CN": "腾讯云 SecretKey(API Key 那一栏填的是 SecretId)",
|
||||
},
|
||||
ModelTypes: rerankOnly,
|
||||
Secret: true,
|
||||
},
|
||||
{
|
||||
// RunRerank is only published in ap-beijing and ap-guangzhou.
|
||||
|
|
@ -129,7 +132,19 @@ func init() {
|
|||
ModelTypes: rerankOnly,
|
||||
},
|
||||
},
|
||||
RerankAPI: api.RerankTencentLKEAP,
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// RunRerank takes at most 60 documents, and Query plus Docs
|
||||
// together at most 2000 characters.
|
||||
MaxDocuments: catalog.Ptr(60),
|
||||
MaxRequestChars: catalog.Ptr(2000),
|
||||
// Batches went out one at a time before the shared batching
|
||||
// layer existed. A 2000-character budget splits a large
|
||||
// candidate set into many requests, so the default fan-out of
|
||||
// four would be a new burst against RunRerank.
|
||||
MaxConcurrency: catalog.Ptr(1),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
MaxTokensField: catalog.Ptr("max_tokens"),
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
// Package novita registers the Novita AI gateway.
|
||||
//
|
||||
// Facts (https://docs.novita.ai — novita.ai/docs/* redirects there):
|
||||
// - chat, embedding and VLM share https://api.novita.ai/openai/v1;
|
||||
// /v1/rerank, /v1/completions, /v1/batches and /v1/files sit alongside;
|
||||
// - chat, embedding, VLM and rerank share https://api.novita.ai/openai/v1;
|
||||
// /v1/completions, /v1/batches and /v1/files sit alongside;
|
||||
// - rerank is the Cohere shape — {model, query, documents, top_n} answering
|
||||
// results[].{index, relevance_score, document.text} — so it needs no
|
||||
// vendor settings beyond the type being open
|
||||
// (https://docs.novita.ai/api-reference/model-apis-llm-create-rerank).
|
||||
// No ceiling on the document count is documented, and the reference
|
||||
// lists no model ids, so none are shipped: the operator names the model;
|
||||
// - output cap is `max_tokens`, documented as required;
|
||||
// `max_completion_tokens` is never mentioned;
|
||||
// - the thinking switch is a **top-level** `enable_thinking` boolean
|
||||
|
|
@ -59,11 +65,13 @@ func init() {
|
|||
DefaultBaseURLs: map[types.ModelType]string{
|
||||
types.ModelTypeKnowledgeQA: BaseURL,
|
||||
types.ModelTypeEmbedding: BaseURL,
|
||||
types.ModelTypeRerank: BaseURL,
|
||||
types.ModelTypeVLLM: BaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
|
|
|
|||
|
|
@ -98,7 +98,17 @@ func init() {
|
|||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
},
|
||||
RerankAPI: api.RerankNIM,
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// rankings[].logit is unbounded and routinely negative, so the
|
||||
// caller must be told this is not a 0..1 relevance score.
|
||||
ScoreScale: catalog.Ptr(api.ScoreLogit),
|
||||
// truncate defaults to NONE upstream, which fails the request on
|
||||
// an over-long passage instead of cutting it.
|
||||
Truncate: catalog.Ptr("END"),
|
||||
MaxDocuments: catalog.Ptr(512),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
MaxTokensField: catalog.Ptr("max_tokens"),
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatChatTemplateKwargs),
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
{"id": "minimax/minimax-m3", "name": "MiniMax M3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 512000, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.06}},
|
||||
{"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}},
|
||||
{"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
|
||||
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"}
|
||||
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
|
||||
{"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,11 +67,13 @@ func init() {
|
|||
DefaultBaseURLs: map[types.ModelType]string{
|
||||
types.ModelTypeKnowledgeQA: BaseURL,
|
||||
types.ModelTypeEmbedding: BaseURL,
|
||||
types.ModelTypeRerank: BaseURL,
|
||||
types.ModelTypeVLLM: BaseURL,
|
||||
},
|
||||
ModelTypes: []types.ModelType{
|
||||
types.ModelTypeKnowledgeQA,
|
||||
types.ModelTypeEmbedding,
|
||||
types.ModelTypeRerank,
|
||||
types.ModelTypeVLLM,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
|
|
|
|||
|
|
@ -101,6 +101,13 @@ func init() {
|
|||
types.ModelTypeVLLM,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// "文本数量不超过64"; query "长度不超过1600个字符"; each document
|
||||
// "长度不超过4096个字符".
|
||||
MaxDocuments: catalog.Ptr(64),
|
||||
MaxQueryChars: catalog.Ptr(1600),
|
||||
MaxDocumentChars: catalog.Ptr(4096),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
// Both spellings are accepted; max_completion_tokens is the
|
||||
// one that also covers the thinking chain.
|
||||
|
|
|
|||
|
|
@ -90,6 +90,16 @@ func TestEveryCatalogEntryResolves(t *testing.T) {
|
|||
modelType = types.ModelTypeKnowledgeQA
|
||||
}
|
||||
r, err := catalog.Resolve(catalog.Ref{Provider: v.ID, Model: name, ModelType: modelType})
|
||||
// An entry may declare that this build cannot serve it — a vendor
|
||||
// whose second rerank dialect has no protocol package. Refusing is
|
||||
// the point: the alternative is a request shaped for the wrong
|
||||
// protocol. Such an entry must refuse, and must do it with a reason.
|
||||
if bytes.Contains(m.Compat, []byte("unsupported_reason")) {
|
||||
if err == nil {
|
||||
t.Errorf("%s/%s: declares unsupported_reason but still resolves", id, name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("%s/%s: resolve: %v", id, name, err)
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -150,8 +150,11 @@ func init() {
|
|||
Type: "password",
|
||||
Required: true,
|
||||
Placeholder: "Volcengine IAM secret key (the API Key field holds the access key)",
|
||||
ModelTypes: rerankOnly,
|
||||
Secret: true,
|
||||
Placeholders: map[string]string{
|
||||
"zh-CN": "火山引擎 IAM Secret Access Key(API Key 那一栏填的是 Access Key ID)",
|
||||
},
|
||||
ModelTypes: rerankOnly,
|
||||
Secret: true,
|
||||
},
|
||||
{
|
||||
Key: "region",
|
||||
|
|
@ -169,10 +172,18 @@ func init() {
|
|||
Type: "string",
|
||||
Default: "Whether the Document answers the Query or matches the content retrieval intent",
|
||||
Placeholder: "Instruction passed to the rerank model",
|
||||
ModelTypes: rerankOnly,
|
||||
Placeholders: map[string]string{
|
||||
"zh-CN": "传给重排模型的 instruction",
|
||||
},
|
||||
ModelTypes: rerankOnly,
|
||||
},
|
||||
},
|
||||
RerankAPI: api.RerankVolcengineKnowledge,
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
MaxDocuments: catalog.Ptr(50),
|
||||
MaxConcurrency: catalog.Ptr(4),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
|
||||
SupportsReasoningEffort: catalog.Ptr(true),
|
||||
|
|
|
|||
|
|
@ -90,6 +90,13 @@ func init() {
|
|||
types.ModelTypeVLLM,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
Path: catalog.Ptr("/api/v1/rerank"),
|
||||
// The only rerank vendor that has ever had a client deadline
|
||||
// here. Without it a hung endpoint holds the retrieval stage
|
||||
// for as long as the caller's context allows.
|
||||
RequestTimeout: catalog.Ptr(60),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
SupportsMultiContent: catalog.Ptr(false),
|
||||
PromptCacheAccounting: catalog.Ptr(false),
|
||||
|
|
|
|||
|
|
@ -109,6 +109,14 @@ func init() {
|
|||
types.ModelTypeVLLM,
|
||||
},
|
||||
Compat: catalog.VendorCompat{
|
||||
Rerank: catalog.RerankCompat{
|
||||
// The reference gives 最大长度为 4096 字符 for the query and for each
|
||||
// document, and caps documents at 128 per request.
|
||||
SendReturnDocs: catalog.Ptr(true),
|
||||
MaxDocuments: catalog.Ptr(128),
|
||||
MaxQueryChars: catalog.Ptr(4096),
|
||||
MaxDocumentChars: catalog.Ptr(4096),
|
||||
},
|
||||
OpenAICompletions: catalog.OpenAICompletionsCompat{
|
||||
MaxTokensField: catalog.Ptr("max_tokens"),
|
||||
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
|
||||
|
|
|
|||
Loading…
Reference in New Issue