From f8daa8d785cd847fe5aa514f3d83b8d49d70ac16 Mon Sep 17 00:00:00 2001 From: dzc-bit <210145775+dzc-bit@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:19:05 +0800 Subject: [PATCH] fix(retrieval): floor ES script_score cosine at 0 to stop 400s on negative similarity Signed-off-by: dzc-bit <210145775+dzc-bit@users.noreply.github.com> --- .../retriever/elasticsearch/v7/repository.go | 13 ++++- .../v7/vector_score_script_test.go | 52 ++++++++++++++++++ .../retriever/elasticsearch/v8/repository.go | 53 ++++++++++++------- .../v8/vector_score_script_test.go | 49 +++++++++++++++++ 4 files changed, 148 insertions(+), 19 deletions(-) create mode 100644 internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go create mode 100644 internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index 75548346..eee56bc2 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -627,6 +627,17 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context, }, nil } +// vectorScoreScriptSource scores every document by its cosine similarity to the +// query vector, floored at 0. Lucene rejects negative final script_score values +// ("script_score script returned an invalid score ... Must be a non-negative +// score!"), so an unclamped negative cosine — which occurs whenever any stored +// vector points away from the query — fails the entire search request with a +// 400 (all shards failed) instead of merely ranking that document last. The +// floor keeps the score in the [0, 1] range the shared retriever score +// normalizer documents for this engine; documents clamped to 0 fall below any +// positive min_score threshold. +var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)" + // buildVectorSearchQuery builds the vector search query JSON func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context, params typesLocal.RetrieveParams, @@ -651,7 +662,7 @@ func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context, }, }, "script": map[string]interface{}{ - "source": "cosineSimilarity(params.query_vector,'embedding')", + "source": vectorScoreScriptSource, "params": map[string]interface{}{ "query_vector": params.Embedding, }, diff --git a/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go b/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go new file mode 100644 index 00000000..070e97b0 --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go @@ -0,0 +1,52 @@ +package v7 + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + typesLocal "github.com/Tencent/WeKnora/internal/types" +) + +// Regression test for #3156: the script_score source must floor the cosine +// similarity at 0. An unclamped negative cosine makes Lucene reject the whole +// search request ("script_score script returned an invalid score ... Must be a +// non-negative score!"), turning one dissimilar document into a full vector +// retrieval outage (all shards failed). +func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) { + require.Equal(t, + "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)", + vectorScoreScriptSource) +} + +// The marshaled query body must carry the clamped script source end to end. +func TestBuildVectorSearchQueryClampsScriptScore(t *testing.T) { + e := &elasticsearchRepository{} + params := typesLocal.RetrieveParams{ + Embedding: []float32{-1, 0}, + Threshold: 0.15, + TopK: 5, + KnowledgeBaseIDs: []string{"kb-1"}, + } + + query, err := e.buildVectorSearchQuery(context.Background(), params) + require.NoError(t, err) + + var body struct { + Query struct { + ScriptScore struct { + Script struct { + Source string `json:"source"` + } `json:"script"` + MinScore float64 `json:"min_score"` + } `json:"script_score"` + } `json:"query"` + } + require.NoError(t, json.Unmarshal([]byte(query), &body)) + require.Equal(t, + "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)", + body.Query.ScriptScore.Script.Source) + require.InDelta(t, 0.15, body.Query.ScriptScore.MinScore, 1e-6) +} diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index 57ce52ad..63134307 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -403,6 +403,39 @@ func (e *elasticsearchRepository) Retrieve(ctx context.Context, return nil, err } +// vectorScoreScriptSource scores every document by its cosine similarity to the +// query vector, floored at 0. Lucene rejects negative final script_score values +// ("script_score script returned an invalid score ... Must be a non-negative +// score!"), so an unclamped negative cosine — which occurs whenever any stored +// vector points away from the query — fails the entire search request with a +// 400 (all shards failed) instead of merely ranking that document last. The +// floor keeps the score in the [0, 1] range the shared retriever score +// normalizer documents for this engine; documents clamped to 0 fall below any +// positive min_score threshold. +var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)" + +// buildVectorScriptScoreQuery wraps the cosine-similarity scoring script in a +// script_score query over the request's base filter conditions. +func (e *elasticsearchRepository) buildVectorScriptScoreQuery( + params typesLocal.RetrieveParams, +) (*types.ScriptScoreQuery, error) { + queryVectorJSON, err := json.Marshal(params.Embedding) + if err != nil { + return nil, fmt.Errorf("failed to marshal query embedding: %w", err) + } + minScore := float32(params.Threshold) + return &types.ScriptScoreQuery{ + Query: types.Query{Bool: &types.BoolQuery{Filter: e.getBaseConds(params)}}, + Script: types.Script{ + Source: &vectorScoreScriptSource, + Params: map[string]json.RawMessage{ + "query_vector": json.RawMessage(queryVectorJSON), + }, + }, + MinScore: &minScore, + }, nil +} + // VectorRetrieve performs vector similarity search using cosine similarity // Returns a slice of RetrieveResult containing matching documents func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context, @@ -412,26 +445,10 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context, log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", len(params.Embedding), params.TopK, params.Threshold) - filter := e.getBaseConds(params) - - // Build script scoring query with cosine similarity - queryVectorJSON, err := json.Marshal(params.Embedding) + scriptScore, err := e.buildVectorScriptScoreQuery(params) if err != nil { log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err) - return nil, fmt.Errorf("failed to marshal query embedding: %w", err) - } - - scoreSource := "cosineSimilarity(params.query_vector, 'embedding')" - minScore := float32(params.Threshold) - scriptScore := &types.ScriptScoreQuery{ - Query: types.Query{Bool: &types.BoolQuery{Filter: filter}}, - Script: types.Script{ - Source: &scoreSource, - Params: map[string]json.RawMessage{ - "query_vector": json.RawMessage(queryVectorJSON), - }, - }, - MinScore: &minScore, + return nil, err } // Exclude embedding field from source to reduce response size sourceFilter := &types.SourceFilter{ diff --git a/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go b/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go new file mode 100644 index 00000000..9cafdbe7 --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go @@ -0,0 +1,49 @@ +package v8 + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + typesLocal "github.com/Tencent/WeKnora/internal/types" +) + +// Regression test for #3156: the script_score source must floor the cosine +// similarity at 0. An unclamped negative cosine makes Lucene reject the whole +// search request ("script_score script returned an invalid score ... Must be a +// non-negative score!"), turning one dissimilar document into a full vector +// retrieval outage (all shards failed). +func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) { + require.Equal(t, + "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)", + vectorScoreScriptSource) +} + +func TestBuildVectorScriptScoreQuery(t *testing.T) { + e := &elasticsearchRepository{} + params := typesLocal.RetrieveParams{ + Embedding: []float32{1, 0}, + Threshold: 0.15, + TopK: 10, + KnowledgeBaseIDs: []string{"kb-1"}, + } + + q, err := e.buildVectorScriptScoreQuery(params) + require.NoError(t, err) + require.NotNil(t, q) + + require.NotNil(t, q.Script.Source) + require.Equal(t, vectorScoreScriptSource, *q.Script.Source) + + require.NotNil(t, q.MinScore) + require.InDelta(t, 0.15, float64(*q.MinScore), 1e-6) + + require.NotNil(t, q.Query.Bool) + require.Len(t, q.Query.Bool.Filter, 1) + + var queryVector []float32 + require.NotNil(t, q.Script.Params["query_vector"]) + require.NoError(t, json.Unmarshal(q.Script.Params["query_vector"], &queryVector)) + require.Equal(t, []float32{1, 0}, queryVector) +}