fix(retrieval): floor ES script_score cosine at 0 to stop 400s on negative similarity

Signed-off-by: dzc-bit <210145775+dzc-bit@users.noreply.github.com>
main
dzc-bit 2026-09-10 16:19:05 +08:00 committed by lyingbug
parent a84f866892
commit f8daa8d785
4 changed files with 148 additions and 19 deletions

View File

@ -627,6 +627,17 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
}, nil
}
// vectorScoreScriptSource scores every document by its cosine similarity to the
// query vector, floored at 0. Lucene rejects negative final script_score values
// ("script_score script returned an invalid score ... Must be a non-negative
// score!"), so an unclamped negative cosine — which occurs whenever any stored
// vector points away from the query — fails the entire search request with a
// 400 (all shards failed) instead of merely ranking that document last. The
// floor keeps the score in the [0, 1] range the shared retriever score
// normalizer documents for this engine; documents clamped to 0 fall below any
// positive min_score threshold.
var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)"
// buildVectorSearchQuery builds the vector search query JSON
func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
params typesLocal.RetrieveParams,
@ -651,7 +662,7 @@ func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
},
},
"script": map[string]interface{}{
"source": "cosineSimilarity(params.query_vector,'embedding')",
"source": vectorScoreScriptSource,
"params": map[string]interface{}{
"query_vector": params.Embedding,
},

View File

@ -0,0 +1,52 @@
package v7
import (
"context"
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
typesLocal "github.com/Tencent/WeKnora/internal/types"
)
// Regression test for #3156: the script_score source must floor the cosine
// similarity at 0. An unclamped negative cosine makes Lucene reject the whole
// search request ("script_score script returned an invalid score ... Must be a
// non-negative score!"), turning one dissimilar document into a full vector
// retrieval outage (all shards failed).
func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {
require.Equal(t,
"Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",
vectorScoreScriptSource)
}
// The marshaled query body must carry the clamped script source end to end.
func TestBuildVectorSearchQueryClampsScriptScore(t *testing.T) {
e := &elasticsearchRepository{}
params := typesLocal.RetrieveParams{
Embedding: []float32{-1, 0},
Threshold: 0.15,
TopK: 5,
KnowledgeBaseIDs: []string{"kb-1"},
}
query, err := e.buildVectorSearchQuery(context.Background(), params)
require.NoError(t, err)
var body struct {
Query struct {
ScriptScore struct {
Script struct {
Source string `json:"source"`
} `json:"script"`
MinScore float64 `json:"min_score"`
} `json:"script_score"`
} `json:"query"`
}
require.NoError(t, json.Unmarshal([]byte(query), &body))
require.Equal(t,
"Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",
body.Query.ScriptScore.Script.Source)
require.InDelta(t, 0.15, body.Query.ScriptScore.MinScore, 1e-6)
}

View File

@ -403,6 +403,39 @@ func (e *elasticsearchRepository) Retrieve(ctx context.Context,
return nil, err
}
// vectorScoreScriptSource scores every document by its cosine similarity to the
// query vector, floored at 0. Lucene rejects negative final script_score values
// ("script_score script returned an invalid score ... Must be a non-negative
// score!"), so an unclamped negative cosine — which occurs whenever any stored
// vector points away from the query — fails the entire search request with a
// 400 (all shards failed) instead of merely ranking that document last. The
// floor keeps the score in the [0, 1] range the shared retriever score
// normalizer documents for this engine; documents clamped to 0 fall below any
// positive min_score threshold.
var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)"
// buildVectorScriptScoreQuery wraps the cosine-similarity scoring script in a
// script_score query over the request's base filter conditions.
func (e *elasticsearchRepository) buildVectorScriptScoreQuery(
params typesLocal.RetrieveParams,
) (*types.ScriptScoreQuery, error) {
queryVectorJSON, err := json.Marshal(params.Embedding)
if err != nil {
return nil, fmt.Errorf("failed to marshal query embedding: %w", err)
}
minScore := float32(params.Threshold)
return &types.ScriptScoreQuery{
Query: types.Query{Bool: &types.BoolQuery{Filter: e.getBaseConds(params)}},
Script: types.Script{
Source: &vectorScoreScriptSource,
Params: map[string]json.RawMessage{
"query_vector": json.RawMessage(queryVectorJSON),
},
},
MinScore: &minScore,
}, nil
}
// VectorRetrieve performs vector similarity search using cosine similarity
// Returns a slice of RetrieveResult containing matching documents
func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
@ -412,26 +445,10 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
len(params.Embedding), params.TopK, params.Threshold)
filter := e.getBaseConds(params)
// Build script scoring query with cosine similarity
queryVectorJSON, err := json.Marshal(params.Embedding)
scriptScore, err := e.buildVectorScriptScoreQuery(params)
if err != nil {
log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err)
return nil, fmt.Errorf("failed to marshal query embedding: %w", err)
}
scoreSource := "cosineSimilarity(params.query_vector, 'embedding')"
minScore := float32(params.Threshold)
scriptScore := &types.ScriptScoreQuery{
Query: types.Query{Bool: &types.BoolQuery{Filter: filter}},
Script: types.Script{
Source: &scoreSource,
Params: map[string]json.RawMessage{
"query_vector": json.RawMessage(queryVectorJSON),
},
},
MinScore: &minScore,
return nil, err
}
// Exclude embedding field from source to reduce response size
sourceFilter := &types.SourceFilter{

View File

@ -0,0 +1,49 @@
package v8
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
typesLocal "github.com/Tencent/WeKnora/internal/types"
)
// Regression test for #3156: the script_score source must floor the cosine
// similarity at 0. An unclamped negative cosine makes Lucene reject the whole
// search request ("script_score script returned an invalid score ... Must be a
// non-negative score!"), turning one dissimilar document into a full vector
// retrieval outage (all shards failed).
func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {
require.Equal(t,
"Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)",
vectorScoreScriptSource)
}
func TestBuildVectorScriptScoreQuery(t *testing.T) {
e := &elasticsearchRepository{}
params := typesLocal.RetrieveParams{
Embedding: []float32{1, 0},
Threshold: 0.15,
TopK: 10,
KnowledgeBaseIDs: []string{"kb-1"},
}
q, err := e.buildVectorScriptScoreQuery(params)
require.NoError(t, err)
require.NotNil(t, q)
require.NotNil(t, q.Script.Source)
require.Equal(t, vectorScoreScriptSource, *q.Script.Source)
require.NotNil(t, q.MinScore)
require.InDelta(t, 0.15, float64(*q.MinScore), 1e-6)
require.NotNil(t, q.Query.Bool)
require.Len(t, q.Query.Bool.Filter, 1)
var queryVector []float32
require.NotNil(t, q.Script.Params["query_vector"])
require.NoError(t, json.Unmarshal(q.Script.Params["query_vector"], &queryVector))
require.Equal(t, []float32{1, 0}, queryVector)
}