diff --git a/config/prompt_templates/agent_system_prompt.yaml b/config/prompt_templates/agent_system_prompt.yaml index 85cc8e52..e0dd221d 100644 --- a/config/prompt_templates/agent_system_prompt.yaml +++ b/config/prompt_templates/agent_system_prompt.yaml @@ -45,7 +45,7 @@ templates: You are WeKnora, an assistant developed by Tencent, specializing in questions and work involving the user's knowledge bases. When the task calls for knowledge-base evidence: - Choose a relevant source from the current runtime directory and use a search or reader supported by its capabilities. A knowledge base's states what it is for and its (gist, topics, typical questions) states what it actually contains; when several are bound, prefer the ones whose profile matches the question and pass their ids to the search tool instead of searching all of them. - - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase rather than repeating an unproductive query. + - Start with search_knowledge: the default hybrid mode covers most questions; use mode=keyword for exact terms, error messages, or identifiers, and mode=semantic for paraphrased or conceptual questions. In hybrid and semantic mode write the query as a complete question, not a keyword list. Rephrase an unproductive query; for exact terms, error messages, or identifiers switch to mode=keyword rather than repeating hybrid or semantic. - Use read_document(id=dN) when a returned passage is incomplete: page with offset, or pass query to find a passage inside the document. Full content already returned for the task does not require another read. list_documents shows what a knowledge base contains when the search surface is unclear. - For exact quotes, numbers, or code, verify the underlying source passage rather than relying on document metadata or a generated overview. Apply the runtime source-selection rules before this default retrieval workflow. Other requested sources and deliverables are part of the user's task. diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.go index 58693ffa..933a67d1 100644 --- a/internal/agent/tools/search_knowledge.go +++ b/internal/agent/tools/search_knowledge.go @@ -41,11 +41,12 @@ var searchKnowledgeTool = BaseTool{ "names the subject and what you want to know (\"Why does Echo use open-weight models to cut cost?\"), " + "not a keyword list (\"Echo open-weight cost\"): results are ranked by a relevance model that scores " + "meaning. In keyword mode write the exact terms. Pass knowledge_base_ids to focus on the bases whose " + - "profile fits the question. When nothing passes the relevance check, rephrase the question; switching " + - "mode alone does not help.\n" + + "profile fits the question. When nothing passes the relevance check, rephrase with terms the documents " + + "would use; repeating the same query in another mode usually does not help. For identifiers, error " + + "codes or exact names retry with mode=keyword.\n" + "Every chunk carries a cN handle and belongs to a dN document. Use read_document(id=dN) to read the " + - "surrounding context or the whole document. Searches match chunk text only; to find a document by its " + - "title or file name use list_documents(keyword=...).", + "surrounding context or the whole document. Retrieval matches chunk text; the relevance model also " + + "sees the document title. To find a document by its title or file name use list_documents(keyword=...).", schema: json.RawMessage(`{ "type": "object", "properties": { @@ -305,16 +306,17 @@ func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage) // emptySearchStatement describes an empty result, including any mode // fallback, so the model does not read "no semantic neighbours" as "the // exact term does not occur". When retrieval found candidates and the rerank -// model rejected all of them, it says so: every mode shares that relevance -// check, so the model should rephrase instead of cycling through modes. +// model rejected all of them, it says so: repeating the same query in +// another mode is usually wasted, but keyword mode can still surface a +// different candidate set for identifiers and exact terms. func emptySearchStatement(query string, data map[string]interface{}, kbCount int) string { mode, _ := data["mode"].(string) msg := fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).", query, mode, kbCount) if rejected, _ := data["rerank_rejected"].(int); rejected > 0 { msg = fmt.Sprintf("No chunk passed the relevance check for %q (mode=%s) in %d knowledge base(s): "+ "retrieval found %d candidate chunks but the relevance model scored none of them as answering the "+ - "query. Every mode applies the same check, so switching mode will not help; rephrase as a complete "+ - "question that names the subject, or use terms the documents themselves would use.", + "query. Repeating the same query in another mode usually will not help; rephrase with terms the "+ + "documents would use. For identifiers, error codes or exact names retry with mode=keyword.", query, mode, kbCount, rejected) } fallbacks, _ := data["mode_fallbacks"].([]map[string]interface{}) diff --git a/internal/agent/tools/search_knowledge_rerank_test.go b/internal/agent/tools/search_knowledge_rerank_test.go index 918efab2..3031643e 100644 --- a/internal/agent/tools/search_knowledge_rerank_test.go +++ b/internal/agent/tools/search_knowledge_rerank_test.go @@ -2,7 +2,9 @@ package tools import ( "context" + "encoding/json" "errors" + "strings" "testing" "github.com/Tencent/WeKnora/internal/config" @@ -220,6 +222,67 @@ func TestRerankScores_prefixDocumentTitle(t *testing.T) { } } +// Title is shared by every chunk of a document, so a passage whose body never +// names the query subject still goes to the reranker as title + chunk. +func TestRerankPassage_prefixesTitleWhenChunkOmitsSubject(t *testing.T) { + t.Parallel() + got := (&SearchKnowledgeTool{}).rerankPassage(context.Background(), &types.SearchResult{ + Content: "installation prerequisites and docker compose flags", + KnowledgeTitle: "Show HN: Echo – Fable-level results at 1/3 the cost using open-weight models", + ChunkType: string(types.ChunkTypeText), + }) + if !strings.HasPrefix(got, "Show HN: Echo") { + t.Fatalf("expected document title prefix, got %q", got) + } + if !strings.Contains(got, "installation prerequisites") { + t.Fatalf("chunk body must still be present: %q", got) + } +} + +// Execute must surface rerank_rejected through Data and the empty statement +// when the model scores every retrieved candidate below the fallback floor. +func TestExecuteReportsRerankRejection(t *testing.T) { + t.Parallel() + model := &stubReranker{scores: []float64{0.10, 0.04}} + svc := &stubKnowledgeBaseService{ + results: []*types.SearchResult{ + { + ID: "c1", Content: "comments section", KnowledgeID: "d1", KnowledgeBaseID: "kb-1", + KnowledgeTitle: "Show HN: Echo", Score: 0.8, + }, + { + ID: "c2", Content: "unrelated notes", KnowledgeID: "d2", KnowledgeBaseID: "kb-1", + KnowledgeTitle: "Other", Score: 0.7, + }, + }, + } + tool := NewSearchKnowledgeTool( + svc, nil, nil, + types.SearchTargets{{ + Type: types.SearchTargetTypeKnowledgeBase, KnowledgeBaseID: "kb-1", TenantID: 1, + }}, + model, + &config.Config{Conversation: &config.ConversationConfig{RerankThreshold: 0.3}}, + ) + res, err := tool.Execute(context.Background(), json.RawMessage( + `{"query":"Why does Echo use open-weight models to cut cost?"}`, + )) + if err != nil || res == nil || !res.Success { + t.Fatalf("Execute: res=%+v err=%v", res, err) + } + if got, _ := res.Data["rerank_rejected"].(int); got != 2 { + t.Fatalf("rerank_rejected = %v, want 2; data=%+v", res.Data["rerank_rejected"], res.Data) + } + if !strings.Contains(res.Output, "found 2 candidate chunks") || + !strings.Contains(res.Output, "Repeating the same query") || + !strings.Contains(res.Output, "mode=keyword") { + t.Fatalf("empty statement = %q", res.Output) + } + if strings.Contains(res.Output, "switching mode will not help") { + t.Fatalf("statement must not forbid a keyword retry: %q", res.Output) + } +} + func TestRerankThreshold_default(t *testing.T) { t.Parallel() tool := &SearchKnowledgeTool{} diff --git a/internal/agent/tools/search_knowledge_test.go b/internal/agent/tools/search_knowledge_test.go index 1521591a..3c73329a 100644 --- a/internal/agent/tools/search_knowledge_test.go +++ b/internal/agent/tools/search_knowledge_test.go @@ -89,15 +89,19 @@ func TestResolveKBSearchModesFallsBackPerKnowledgeBase(t *testing.T) { } // When the rerank model rejected every candidate, the statement must not -// read as "nothing was found": switching mode cannot get past that check. +// read as "nothing was found": repeating the same query in another mode is +// usually wasted, but keyword mode remains the retry for exact terms. func TestEmptySearchStatementReportsRerankRejection(t *testing.T) { data := map[string]interface{}{"mode": SearchModeHybrid, "rerank_rejected": 18} msg := emptySearchStatement("Echo open-weight cost", data, 20) - for _, want := range []string{"found 18 candidate chunks", "switching mode will not help", "complete question"} { + for _, want := range []string{"found 18 candidate chunks", "Repeating the same query", "mode=keyword"} { if !strings.Contains(msg, want) { t.Fatalf("statement missing %q: %q", want, msg) } } + if strings.Contains(msg, "switching mode will not help") { + t.Fatalf("statement must not forbid switching to keyword: %q", msg) + } if plain := emptySearchStatement("x", map[string]interface{}{"mode": SearchModeHybrid}, 1); !strings.HasPrefix( plain, "No matching chunks") { t.Fatalf("no rerank rejection keeps the plain statement: %q", plain)