fix(agent): allow keyword retry after search_knowledge rerank rejection
Repeating the same query in another mode is usually wasted, but keyword mode can still recall identifiers that hybrid never surfaced. Stop telling the model that switching mode cannot help.main
parent
62bb9da270
commit
2a6e5dcbb2
|
|
@ -45,7 +45,7 @@ templates:
|
|||
You are WeKnora, an assistant developed by Tencent, specializing in questions and work involving the user's knowledge bases.
|
||||
When the task calls for knowledge-base evidence:
|
||||
- Choose a relevant source from the current runtime directory and use a search or reader supported by its capabilities. A knowledge base's <description> states what it is for and its <generated_profile> (gist, topics, typical questions) states what it actually contains; when several are bound, prefer the ones whose profile matches the question and pass their ids to the search tool instead of searching all of them.
|
||||
- Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase rather than repeating an unproductive query.
|
||||
- Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase an unproductive query; for exact terms, error messages, or identifiers switch to mode=keyword rather than repeating hybrid or semantic.
|
||||
- Use read_document(id=dN) when a returned passage is incomplete: page with offset, or pass query to find a passage inside the document. Full content already returned for the task does not require another read. list_documents shows what a knowledge base contains when the search surface is unclear.
|
||||
- For exact quotes, numbers, or code, verify the underlying source passage rather than relying on document metadata or a generated overview.
|
||||
Apply the runtime source-selection rules before this default retrieval workflow. Other requested sources and deliverables are part of the user's task.
|
||||
|
|
|
|||
|
|
@ -41,11 +41,12 @@ var searchKnowledgeTool = BaseTool{
|
|||
"names the subject and what you want to know (\"Why does Echo use open-weight models to cut cost?\"), " +
|
||||
"not a keyword list (\"Echo open-weight cost\"): results are ranked by a relevance model that scores " +
|
||||
"meaning. In keyword mode write the exact terms. Pass knowledge_base_ids to focus on the bases whose " +
|
||||
"profile fits the question. When nothing passes the relevance check, rephrase the question; switching " +
|
||||
"mode alone does not help.\n" +
|
||||
"profile fits the question. When nothing passes the relevance check, rephrase with terms the documents " +
|
||||
"would use; repeating the same query in another mode usually does not help. For identifiers, error " +
|
||||
"codes or exact names retry with mode=keyword.\n" +
|
||||
"Every chunk carries a cN handle and belongs to a dN document. Use read_document(id=dN) to read the " +
|
||||
"surrounding context or the whole document. Searches match chunk text only; to find a document by its " +
|
||||
"title or file name use list_documents(keyword=...).",
|
||||
"surrounding context or the whole document. Retrieval matches chunk text; the relevance model also " +
|
||||
"sees the document title. To find a document by its title or file name use list_documents(keyword=...).",
|
||||
schema: json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -305,16 +306,17 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage)
|
|||
// emptySearchStatement describes an empty result, including any mode
|
||||
// fallback, so the model does not read "no semantic neighbours" as "the
|
||||
// exact term does not occur". When retrieval found candidates and the rerank
|
||||
// model rejected all of them, it says so: every mode shares that relevance
|
||||
// check, so the model should rephrase instead of cycling through modes.
|
||||
// model rejected all of them, it says so: repeating the same query in
|
||||
// another mode is usually wasted, but keyword mode can still surface a
|
||||
// different candidate set for identifiers and exact terms.
|
||||
func emptySearchStatement(query string, data map[string]interface{}, kbCount int) string {
|
||||
mode, _ := data["mode"].(string)
|
||||
msg := fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).", query, mode, kbCount)
|
||||
if rejected, _ := data["rerank_rejected"].(int); rejected > 0 {
|
||||
msg = fmt.Sprintf("No chunk passed the relevance check for %q (mode=%s) in %d knowledge base(s): "+
|
||||
"retrieval found %d candidate chunks but the relevance model scored none of them as answering the "+
|
||||
"query. Every mode applies the same check, so switching mode will not help; rephrase as a complete "+
|
||||
"question that names the subject, or use terms the documents themselves would use.",
|
||||
"query. Repeating the same query in another mode usually will not help; rephrase with terms the "+
|
||||
"documents would use. For identifiers, error codes or exact names retry with mode=keyword.",
|
||||
query, mode, kbCount, rejected)
|
||||
}
|
||||
fallbacks, _ := data["mode_fallbacks"].([]map[string]interface{})
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ package tools
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
|
|
@ -220,6 +222,67 @@ func TestRerankScores_prefixDocumentTitle(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Title is shared by every chunk of a document, so a passage whose body never
|
||||
// names the query subject still goes to the reranker as title + chunk.
|
||||
func TestRerankPassage_prefixesTitleWhenChunkOmitsSubject(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := (&SearchKnowledgeTool{}).rerankPassage(context.Background(), &types.SearchResult{
|
||||
Content: "installation prerequisites and docker compose flags",
|
||||
KnowledgeTitle: "Show HN: Echo – Fable-level results at 1/3 the cost using open-weight models",
|
||||
ChunkType: string(types.ChunkTypeText),
|
||||
})
|
||||
if !strings.HasPrefix(got, "Show HN: Echo") {
|
||||
t.Fatalf("expected document title prefix, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "installation prerequisites") {
|
||||
t.Fatalf("chunk body must still be present: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute must surface rerank_rejected through Data and the empty statement
|
||||
// when the model scores every retrieved candidate below the fallback floor.
|
||||
func TestExecuteReportsRerankRejection(t *testing.T) {
|
||||
t.Parallel()
|
||||
model := &stubReranker{scores: []float64{0.10, 0.04}}
|
||||
svc := &stubKnowledgeBaseService{
|
||||
results: []*types.SearchResult{
|
||||
{
|
||||
ID: "c1", Content: "comments section", KnowledgeID: "d1", KnowledgeBaseID: "kb-1",
|
||||
KnowledgeTitle: "Show HN: Echo", Score: 0.8,
|
||||
},
|
||||
{
|
||||
ID: "c2", Content: "unrelated notes", KnowledgeID: "d2", KnowledgeBaseID: "kb-1",
|
||||
KnowledgeTitle: "Other", Score: 0.7,
|
||||
},
|
||||
},
|
||||
}
|
||||
tool := NewSearchKnowledgeTool(
|
||||
svc, nil, nil,
|
||||
types.SearchTargets{{
|
||||
Type: types.SearchTargetTypeKnowledgeBase, KnowledgeBaseID: "kb-1", TenantID: 1,
|
||||
}},
|
||||
model,
|
||||
&config.Config{Conversation: &config.ConversationConfig{RerankThreshold: 0.3}},
|
||||
)
|
||||
res, err := tool.Execute(context.Background(), json.RawMessage(
|
||||
`{"query":"Why does Echo use open-weight models to cut cost?"}`,
|
||||
))
|
||||
if err != nil || res == nil || !res.Success {
|
||||
t.Fatalf("Execute: res=%+v err=%v", res, err)
|
||||
}
|
||||
if got, _ := res.Data["rerank_rejected"].(int); got != 2 {
|
||||
t.Fatalf("rerank_rejected = %v, want 2; data=%+v", res.Data["rerank_rejected"], res.Data)
|
||||
}
|
||||
if !strings.Contains(res.Output, "found 2 candidate chunks") ||
|
||||
!strings.Contains(res.Output, "Repeating the same query") ||
|
||||
!strings.Contains(res.Output, "mode=keyword") {
|
||||
t.Fatalf("empty statement = %q", res.Output)
|
||||
}
|
||||
if strings.Contains(res.Output, "switching mode will not help") {
|
||||
t.Fatalf("statement must not forbid a keyword retry: %q", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRerankThreshold_default(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool := &SearchKnowledgeTool{}
|
||||
|
|
|
|||
|
|
@ -89,15 +89,19 @@ func TestResolveKBSearchModesFallsBackPerKnowledgeBase(t *testing.T) {
|
|||
}
|
||||
|
||||
// When the rerank model rejected every candidate, the statement must not
|
||||
// read as "nothing was found": switching mode cannot get past that check.
|
||||
// read as "nothing was found": repeating the same query in another mode is
|
||||
// usually wasted, but keyword mode remains the retry for exact terms.
|
||||
func TestEmptySearchStatementReportsRerankRejection(t *testing.T) {
|
||||
data := map[string]interface{}{"mode": SearchModeHybrid, "rerank_rejected": 18}
|
||||
msg := emptySearchStatement("Echo open-weight cost", data, 20)
|
||||
for _, want := range []string{"found 18 candidate chunks", "switching mode will not help", "complete question"} {
|
||||
for _, want := range []string{"found 18 candidate chunks", "Repeating the same query", "mode=keyword"} {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Fatalf("statement missing %q: %q", want, msg)
|
||||
}
|
||||
}
|
||||
if strings.Contains(msg, "switching mode will not help") {
|
||||
t.Fatalf("statement must not forbid switching to keyword: %q", msg)
|
||||
}
|
||||
if plain := emptySearchStatement("x", map[string]interface{}{"mode": SearchModeHybrid}, 1); !strings.HasPrefix(
|
||||
plain, "No matching chunks") {
|
||||
t.Fatalf("no rerank rejection keeps the plain statement: %q", plain)
|
||||
|
|
|
|||
Loading…
Reference in New Issue