From aac4f1586b4d951efa35c192ed4e0a3230729fee Mon Sep 17 00:00:00 2001 From: Zhang GH Date: Thu, 20 Aug 2026 11:36:38 +0800 Subject: [PATCH] Fix/hybrid search match count normalization (#2750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(hybrid-search): normalize MatchCount once instead of at the truncation site An omitted match_count deserializes to Go's zero value, which reached the truncation step in HybridSearch and sliced the deduplicated chunk list to [:0] — a successful retrieval returned an empty result set. A negative value panicked on that same slice bound and surfaced as a 500. Resolve the value once at the top of HybridSearch via normalizedMatchCount so all three consumers agree, rather than patching the truncation site alone: the over-retrieval floor, the FAQ iterative-retrieval trigger (which never fired on this path because len(chunks) < 0 is false), and the truncation itself. Bound the FAQ iterative path while here, since normalization is what makes it reachable with a large MatchCount: its TopK seeds from MatchCount*3 and doubles five times, previously without limit. It now stops at the pool cap instead of re-issuing an identical query. Co-authored-by: Cursor * refactor(retrieval): share the default retrieval depth as one constant The value 50 was written three times — the over-retrieval floor in HybridSearch, the MatchCount fallback added alongside it, and RetrievalConfig.GetEffectiveEmbeddingTopK — with nothing recording that the three are meant to agree. Anyone tuning one would silently desync the others. Promote it to types.DefaultRetrievalTopK. No behavior change. Co-authored-by: Cursor --------- Co-authored-by: drakezhang Co-authored-by: Cursor --- .../service/knowledgebase_search.go | 53 +++++++++++--- .../service/knowledgebase_search_faq.go | 16 +++- .../knowledgebase_search_matchcount_test.go | 73 +++++++++++++++++++ internal/types/retrieval_config.go | 8 +- .../02-architecture/04-rag-pipeline.md | 4 +- website-docs/03-features/17-faq.md | 2 +- 6 files changed, 137 insertions(+), 19 deletions(-) create mode 100644 internal/application/service/knowledgebase_search_matchcount_test.go diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.go index ef21dbca..c4f7510d 100644 --- a/internal/application/service/knowledgebase_search.go +++ b/internal/application/service/knowledgebase_search.go @@ -12,6 +12,12 @@ import ( secutils "github.com/Tencent/WeKnora/internal/utils" ) +// maxRetrievalPoolSize bounds every retrieval depth derived from MatchCount: +// the over-retrieval pool below and the doubling TopK of the FAQ iterative +// path. Without it a caller-supplied MatchCount scales the vector-store query +// depth without limit. +const maxRetrievalPoolSize = 500 + // GetQueryEmbedding computes the query embedding using the embedding model // associated with the given knowledge base. Callers can pre-compute and reuse // the result across multiple KBs that share the same embedding model to avoid @@ -88,6 +94,10 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, id string, params types.SearchParams, ) ([]*types.SearchResult, error) { + // Normalize once, before anything reads MatchCount. params is a value + // copy, so this stays local to the call. + params.MatchCount = normalizedMatchCount(params.MatchCount) + // Determine the set of KB IDs to search. searchKBIDs := params.KnowledgeBaseIDs if len(searchKBIDs) == 0 { @@ -147,10 +157,11 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, } // Over-retrieval (existing rule, preserved): 5x per-KB matchCount, - // floor of 50, capped at 500 across the whole search. - matchCount := max(params.MatchCount*5, 50) * len(searchKBIDs) - if matchCount > 500 { - matchCount = 500 + // floored at DefaultRetrievalTopK, capped at maxRetrievalPoolSize across + // the whole search. + matchCount := max(params.MatchCount*5, types.DefaultRetrievalTopK) * len(searchKBIDs) + if matchCount > maxRetrievalPoolSize { + matchCount = maxRetrievalPoolSize } // Compute the query embedding once before fan-out and propagate via @@ -248,17 +259,37 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, return nil, err } - // Limit to MatchCount; 0 means "no explicit limit" — use over-retrieval floor (50). - effectiveMatchCount := params.MatchCount - if effectiveMatchCount <= 0 { - effectiveMatchCount = 50 - } - if len(deduplicatedChunks) > effectiveMatchCount { - deduplicatedChunks = deduplicatedChunks[:effectiveMatchCount] + // Truncate to the primary-match cap. MatchCount is guaranteed positive by + // the normalization at the top of this function; the slice bound below + // depends on that. + if len(deduplicatedChunks) > params.MatchCount { + deduplicatedChunks = deduplicatedChunks[:params.MatchCount] } + return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment) } +// normalizedMatchCount resolves the effective primary-match cap for a search. +// +// MatchCount arrives as Go's zero value whenever a caller omits it — JSON +// cannot tell an absent match_count from an explicit 0 — and three consumers +// inside HybridSearch read it: the over-retrieval floor, the FAQ +// iterative-retrieval trigger, and the final truncation. They must agree, so +// the value is resolved here rather than at each use site. A raw 0 truncates +// the deduplicated chunk list to [:0] and empties an otherwise successful +// search; a negative value panics on that same slice bound. +// +// The fallback is shared with RetrievalConfig.GetEffectiveEmbeddingTopK: +// internal callers (chat pipeline, agent tools) feed these results into +// reranking, which trims to RerankTopK afterwards, so a wide candidate pool is +// the right failure mode for them. +func normalizedMatchCount(requested int) int { + if requested <= 0 { + return types.DefaultRetrievalTopK + } + return requested +} + // pickPrimary returns the KB whose ID matches id, or nil if id is not in // scope. Callers map a nil result to NotFound; there is intentionally no // kbs[0] fallback because it would mask caller bugs and could leak an diff --git a/internal/application/service/knowledgebase_search_faq.go b/internal/application/service/knowledgebase_search_faq.go index 124da2c0..e6f28a6f 100644 --- a/internal/application/service/knowledgebase_search_faq.go +++ b/internal/application/service/knowledgebase_search_faq.go @@ -77,8 +77,11 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co ) ([]*types.IndexWithScore, error) { maxIterations := 5 // Start with a larger TopK since we're called when first retrieval wasn't enough - // The first retrieval already used matchCount*3, so start from there - currentTopK := matchCount * 3 + // The first retrieval already used matchCount*3, so start from there. + // matchCount is caller-supplied, so both the seed and the per-iteration + // doubling are bounded by maxRetrievalPoolSize — otherwise a single request + // could drive the vector-store query depth arbitrarily deep. + currentTopK := min(matchCount*3, maxRetrievalPoolSize) uniqueChunks := make(map[string]*types.IndexWithScore) // Cache chunk data to avoid repeated DB queries across iterations chunkDataCache := make(map[string]*types.Chunk) @@ -198,8 +201,13 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co break } - // Increase TopK for next iteration - currentTopK *= 2 + // Increase TopK for next iteration. Once the cap is reached, another + // round would re-issue an identical query, so stop instead. + if currentTopK >= maxRetrievalPoolSize { + logger.Infof(ctx, "Retrieval depth cap %d reached, stopping iteration", maxRetrievalPoolSize) + break + } + currentTopK = min(currentTopK*2, maxRetrievalPoolSize) } // Convert map to slice and sort by score diff --git a/internal/application/service/knowledgebase_search_matchcount_test.go b/internal/application/service/knowledgebase_search_matchcount_test.go new file mode 100644 index 00000000..da8df5f7 --- /dev/null +++ b/internal/application/service/knowledgebase_search_matchcount_test.go @@ -0,0 +1,73 @@ +package service + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Tencent/WeKnora/internal/types" +) + +// TestNormalizedMatchCount pins the contract that a non-positive MatchCount +// resolves to the service-wide retrieval depth. The regression this guards is +// severe and silent: a caller that omitted match_count reached the truncation +// step with 0, which sliced the deduplicated chunk list to [:0] and turned a +// successful retrieval into an empty response. A negative value panicked on +// the same slice bound. +// +// The fallback shares DefaultRetrievalTopK with the over-retrieval floor on +// purpose: with no explicit MatchCount the `*5` amplification in HybridSearch +// collapses, so that floor alone decides the per-retriever depth and a larger +// truncation bound could never yield more results. +func TestNormalizedMatchCount(t *testing.T) { + t.Parallel() + tests := []struct { + name string + requested int + want int + }{ + {name: "omitted arrives as zero", requested: 0, want: types.DefaultRetrievalTopK}, + {name: "negative cannot index a slice", requested: -1, want: types.DefaultRetrievalTopK}, + {name: "explicit value is honored", requested: 3, want: 3}, + {name: "large explicit value is not clamped here", requested: 10000, want: 10000}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, normalizedMatchCount(tt.requested)) + }) + } +} + +// TestIterativeRetrieve_CapsSeedTopK guards the FAQ iterative path against an +// unbounded caller-supplied MatchCount. The seed is MatchCount*3 and doubles +// each round, so without the cap a single request could ask a vector store for +// hundreds of thousands of rows. +func TestIterativeRetrieve_CapsSeedTopK(t *testing.T) { + t.Parallel() + // canned is nil, so the first iteration retrieves nothing and the loop + // breaks — leaving group.TopK at exactly the seed value under test. + empty := &fakeRetrieveEngineService{ + engineType: types.PostgresRetrieverEngineType, + support: []types.RetrieverType{types.VectorRetrieverType}, + } + groups := []*storeGroup{ + { + Engine: buildBoundComposite(t, empty), + BaseParams: vectorParams("q"), + TopK: 50, + KBIDs: []string{"kb-1"}, + }, + } + s := &knowledgeBaseService{} + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + + results, err := s.iterativeRetrieveWithDeduplication(ctx, groups, 100000, "q") + require.NoError(t, err) + assert.Empty(t, results) + assert.Equal(t, maxRetrievalPoolSize, groups[0].TopK, + "seed TopK must be capped at the retrieval pool bound") +} diff --git a/internal/types/retrieval_config.go b/internal/types/retrieval_config.go index f62604a1..fe5ab23b 100644 --- a/internal/types/retrieval_config.go +++ b/internal/types/retrieval_config.go @@ -37,10 +37,16 @@ type RetrievalConfig struct { RRFKeywordWeight float64 `json:"rrf_keyword_weight,omitempty"` } +// DefaultRetrievalTopK is the retrieval depth used when a caller supplies no +// usable TopK / MatchCount. HybridSearch also floors its over-retrieval pool +// at this value, so the fallback and the pool it draws from stay in step by +// construction rather than by two independently maintained literals. +const DefaultRetrievalTopK = 50 + // GetEffectiveEmbeddingTopK returns EmbeddingTopK with a fallback default. func (c *RetrievalConfig) GetEffectiveEmbeddingTopK() int { if c == nil || c.EmbeddingTopK <= 0 { - return 50 + return DefaultRetrievalTopK } return c.EmbeddingTopK } diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.md index 7e57c383..c7dd2ccc 100644 --- a/website-docs/02-architecture/04-rag-pipeline.md +++ b/website-docs/02-architecture/04-rag-pipeline.md @@ -476,7 +476,7 @@ sequenceDiagram `internal/application/service/knowledgebase_search.go` 是所有检索的汇聚点(chat pipeline、Agent 工具、搜索 API 共用): 1. **授权与校验**:批量加载 KB(含跨租户 Organization 共享库),逐库 `authorizeKBAccess`;`validateSameEmbeddingModel` 拒绝跨 embedding 空间的多库检索(wiki/graph 无向量库有豁免)。 -2. **过召回**:`matchCount = max(MatchCount*5, 50) * len(KBs)`,上限 500。 +2. **入参归一化 + 过召回**:`MatchCount <= 0`(调用方未传时 JSON 反序列化即为 0)先经 `normalizedMatchCount` 归一化为 `types.DefaultRetrievalTopK`(50),使过召回下限、FAQ 迭代触发条件、末尾截断三处读到同一个值——否则截断会把结果集切成 `[:0]`,负数还会越界 panic;随后 `matchCount = max(MatchCount*5, 50) * len(KBs)`,上限 `maxRetrievalPoolSize`(500)。 3. **查询向量只算一次**,随 `params.QueryEmbedding` 传播到所有 store 组。 4. **storeGroup 分组**(`knowledgebase_search_storegroup.go`):按 `(VectorStoreID, 属主租户)` 分组;每组经 `retriever.CreateRetrieveEngineForKB` 解析出 `CompositeRetrieveEngine`。`buildRetrievalParams` 按组内每个 KB 的类型路由:FAQ 库走 FAQ 向量索引(`KnowledgeType=faq`,无关键词索引),文档库走默认向量索引 + 关键词索引。 5. **fan-out**(`knowledgebase_search_fanout.go`):单组直查零开销;多组用 `errgroup` 并发(上限 4),每组超时 `MULTI_STORE_RETRIEVE_TIMEOUT_SEC`(默认 30s),all-or-nothing 失败策略;结果跨引擎类型时用 `EngineAwareNormalizer` 把向量分归一化到 [0,1](详见检索引擎文档)。 @@ -484,7 +484,7 @@ sequenceDiagram - 仅向量或仅关键词 → `deduplicateByScore`(按 chunk 保留最高分); - 混合 → **加权 RRF**:`score = vectorWeight/(k+vectorRank) + keywordWeight/(k+keywordRank)`,`k` 与权重来自租户 `RetrievalConfig`(有缺省值),rank 基于各自检索器返回顺序(1-indexed),对分数尺度免疫。 7. **FAQ 命中策略**(`knowledgebase_search_faq.go`,仅 FAQ 类型 KB): - - **迭代检索**:去重后不足 `MatchCount` 且首轮已打满 → 从 `TopK*3` 起最多 5 轮翻倍扩大 TopK 重检索,跨 store 组统一生效,chunk 数据缓存避免重复回表; + - **迭代检索**:去重后不足 `MatchCount` 且首轮已打满 → 从 `TopK*3` 起最多 5 轮翻倍扩大 TopK 重检索,跨 store 组统一生效,chunk 数据缓存避免重复回表;种子与每轮增长均封顶 `maxRetrievalPoolSize`,触顶即停(再迭代只会重发同一个查询); - **负例问题过滤**:查询与 FAQ 的 `NegativeQuestions` 精确匹配(小写去空格)即剔除该条——支持"这个问题不要用这条 FAQ 答"的运营配置。 8. 截断到 `MatchCount` 后 `processSearchResults` 补全 chunk 元数据(管线场景 `SkipContextEnrichment=true`,上下文组装留给 merge 阶段)。 diff --git a/website-docs/03-features/17-faq.md b/website-docs/03-features/17-faq.md index 643fc8ce..42c5d673 100644 --- a/website-docs/03-features/17-faq.md +++ b/website-docs/03-features/17-faq.md @@ -187,7 +187,7 @@ type FAQSearchRequest struct { 1. **混合召回**:查询文本归一化后做向量检索 + BM25 关键词检索,融合去重; 2. **两级标签优先**:`FirstPriorityTagIDs` 命中的条目排最前,其次 `SecondPriorityTagIDs`; 3. **负例过滤**(`filterByNegativeQuestions`):查询文本与某条目的任一反例问完全匹配(小写比较)→ 该条目从结果中剔除。典型场景:用户问"不支持 X 吗",避免返回"支持 X"的条目; -4. **迭代召回**(`applyFAQPostProcessing`):当过滤后的唯一条目数不足 `match_count` 且向量结果打满时触发 `iterativeRetrieveWithDeduplication`——最多迭代 5 次、每次 TopK 翻倍,带去重与负例过滤缓存,无新结果提前终止; +4. **迭代召回**(`applyFAQPostProcessing`):当过滤后的唯一条目数不足 `match_count` 且向量结果打满时触发 `iterativeRetrieveWithDeduplication`——最多迭代 5 次、每次 TopK 翻倍(种子 `MatchCount*3` 与每轮增长均封顶 500,触顶即停),带去重与负例过滤缓存,无新结果提前终止; 5. 结果附带 `score`、`match_type`、`matched_question`(实际命中的是标准问还是哪个相似问),答案按 `answer_strategy`(all / random)返回。 非 FAQ 类型 KB 直接跳过该后处理(`if kb.Type != types.KnowledgeBaseTypeFAQ { return chunks, nil }`),普通混合检索不受影响;agent 检索链在 FAQ 库上同样经过这条后处理路径。