fix(retrieval): floor ES script_score cosine at 0 to stop 400s on negative similarity
Signed-off-by: dzc-bit <210145775+dzc-bit@users.noreply.github.com>main
parent
a84f866892
commit
f8daa8d785
|
|
@ -627,6 +627,17 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
|
|||
}, nil
|
||||
}
|
||||
|
||||
// vectorScoreScriptSource scores every document by its cosine similarity to the
|
||||
// query vector, floored at 0. Lucene rejects negative final script_score values
|
||||
// ("script_score script returned an invalid score ... Must be a non-negative
|
||||
// score!"), so an unclamped negative cosine — which occurs whenever any stored
|
||||
// vector points away from the query — fails the entire search request with a
|
||||
// 400 (all shards failed) instead of merely ranking that document last. The
|
||||
// floor keeps the score in the [0, 1] range the shared retriever score
|
||||
// normalizer documents for this engine; documents clamped to 0 fall below any
|
||||
// positive min_score threshold.
|
||||
var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)"
|
||||
|
||||
// buildVectorSearchQuery builds the vector search query JSON
|
||||
func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
|
|
@ -651,7 +662,7 @@ func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
|
|||
},
|
||||
},
|
||||
"script": map[string]interface{}{
|
||||
"source": "cosineSimilarity(params.query_vector,'embedding')",
|
||||
"source": vectorScoreScriptSource,
|
||||
"params": map[string]interface{}{
|
||||
"query_vector": params.Embedding,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
package v7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
typesLocal "github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Regression test for #3156: the script_score source must floor the cosine
|
||||
// similarity at 0. An unclamped negative cosine makes Lucene reject the whole
|
||||
// search request ("script_score script returned an invalid score ... Must be a
|
||||
// non-negative score!"), turning one dissimilar document into a full vector
|
||||
// retrieval outage (all shards failed).
|
||||
func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {
|
||||
require.Equal(t,
|
||||
"Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",
|
||||
vectorScoreScriptSource)
|
||||
}
|
||||
|
||||
// The marshaled query body must carry the clamped script source end to end.
|
||||
func TestBuildVectorSearchQueryClampsScriptScore(t *testing.T) {
|
||||
e := &elasticsearchRepository{}
|
||||
params := typesLocal.RetrieveParams{
|
||||
Embedding: []float32{-1, 0},
|
||||
Threshold: 0.15,
|
||||
TopK: 5,
|
||||
KnowledgeBaseIDs: []string{"kb-1"},
|
||||
}
|
||||
|
||||
query, err := e.buildVectorSearchQuery(context.Background(), params)
|
||||
require.NoError(t, err)
|
||||
|
||||
var body struct {
|
||||
Query struct {
|
||||
ScriptScore struct {
|
||||
Script struct {
|
||||
Source string `json:"source"`
|
||||
} `json:"script"`
|
||||
MinScore float64 `json:"min_score"`
|
||||
} `json:"script_score"`
|
||||
} `json:"query"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal([]byte(query), &body))
|
||||
require.Equal(t,
|
||||
"Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",
|
||||
body.Query.ScriptScore.Script.Source)
|
||||
require.InDelta(t, 0.15, body.Query.ScriptScore.MinScore, 1e-6)
|
||||
}
|
||||
|
|
@ -403,6 +403,39 @@ func (e *elasticsearchRepository) Retrieve(ctx context.Context,
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// vectorScoreScriptSource scores every document by its cosine similarity to the
|
||||
// query vector, floored at 0. Lucene rejects negative final script_score values
|
||||
// ("script_score script returned an invalid score ... Must be a non-negative
|
||||
// score!"), so an unclamped negative cosine — which occurs whenever any stored
|
||||
// vector points away from the query — fails the entire search request with a
|
||||
// 400 (all shards failed) instead of merely ranking that document last. The
|
||||
// floor keeps the score in the [0, 1] range the shared retriever score
|
||||
// normalizer documents for this engine; documents clamped to 0 fall below any
|
||||
// positive min_score threshold.
|
||||
var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)"
|
||||
|
||||
// buildVectorScriptScoreQuery wraps the cosine-similarity scoring script in a
|
||||
// script_score query over the request's base filter conditions.
|
||||
func (e *elasticsearchRepository) buildVectorScriptScoreQuery(
|
||||
params typesLocal.RetrieveParams,
|
||||
) (*types.ScriptScoreQuery, error) {
|
||||
queryVectorJSON, err := json.Marshal(params.Embedding)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal query embedding: %w", err)
|
||||
}
|
||||
minScore := float32(params.Threshold)
|
||||
return &types.ScriptScoreQuery{
|
||||
Query: types.Query{Bool: &types.BoolQuery{Filter: e.getBaseConds(params)}},
|
||||
Script: types.Script{
|
||||
Source: &vectorScoreScriptSource,
|
||||
Params: map[string]json.RawMessage{
|
||||
"query_vector": json.RawMessage(queryVectorJSON),
|
||||
},
|
||||
},
|
||||
MinScore: &minScore,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// VectorRetrieve performs vector similarity search using cosine similarity
|
||||
// Returns a slice of RetrieveResult containing matching documents
|
||||
func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
|
||||
|
|
@ -412,26 +445,10 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
|
|||
log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
|
||||
len(params.Embedding), params.TopK, params.Threshold)
|
||||
|
||||
filter := e.getBaseConds(params)
|
||||
|
||||
// Build script scoring query with cosine similarity
|
||||
queryVectorJSON, err := json.Marshal(params.Embedding)
|
||||
scriptScore, err := e.buildVectorScriptScoreQuery(params)
|
||||
if err != nil {
|
||||
log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err)
|
||||
return nil, fmt.Errorf("failed to marshal query embedding: %w", err)
|
||||
}
|
||||
|
||||
scoreSource := "cosineSimilarity(params.query_vector, 'embedding')"
|
||||
minScore := float32(params.Threshold)
|
||||
scriptScore := &types.ScriptScoreQuery{
|
||||
Query: types.Query{Bool: &types.BoolQuery{Filter: filter}},
|
||||
Script: types.Script{
|
||||
Source: &scoreSource,
|
||||
Params: map[string]json.RawMessage{
|
||||
"query_vector": json.RawMessage(queryVectorJSON),
|
||||
},
|
||||
},
|
||||
MinScore: &minScore,
|
||||
return nil, err
|
||||
}
|
||||
// Exclude embedding field from source to reduce response size
|
||||
sourceFilter := &types.SourceFilter{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
package v8
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
typesLocal "github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Regression test for #3156: the script_score source must floor the cosine
|
||||
// similarity at 0. An unclamped negative cosine makes Lucene reject the whole
|
||||
// search request ("script_score script returned an invalid score ... Must be a
|
||||
// non-negative score!"), turning one dissimilar document into a full vector
|
||||
// retrieval outage (all shards failed).
|
||||
func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {
|
||||
require.Equal(t,
|
||||
"Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)",
|
||||
vectorScoreScriptSource)
|
||||
}
|
||||
|
||||
func TestBuildVectorScriptScoreQuery(t *testing.T) {
|
||||
e := &elasticsearchRepository{}
|
||||
params := typesLocal.RetrieveParams{
|
||||
Embedding: []float32{1, 0},
|
||||
Threshold: 0.15,
|
||||
TopK: 10,
|
||||
KnowledgeBaseIDs: []string{"kb-1"},
|
||||
}
|
||||
|
||||
q, err := e.buildVectorScriptScoreQuery(params)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, q)
|
||||
|
||||
require.NotNil(t, q.Script.Source)
|
||||
require.Equal(t, vectorScoreScriptSource, *q.Script.Source)
|
||||
|
||||
require.NotNil(t, q.MinScore)
|
||||
require.InDelta(t, 0.15, float64(*q.MinScore), 1e-6)
|
||||
|
||||
require.NotNil(t, q.Query.Bool)
|
||||
require.Len(t, q.Query.Bool.Filter, 1)
|
||||
|
||||
var queryVector []float32
|
||||
require.NotNil(t, q.Script.Params["query_vector"])
|
||||
require.NoError(t, json.Unmarshal(q.Script.Params["query_vector"], &queryVector))
|
||||
require.Equal(t, []float32{1, 0}, queryVector)
|
||||
}
|
||||
Loading…
Reference in New Issue