From c7601b2d2aaf298939b5ce775970c3005044d74a Mon Sep 17 00:00:00 2001 From: MercuialC <946196959@qq.com> Date: Fri, 25 Sep 2026 22:22:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(wiki):=20=E5=A2=9E=E5=8A=A0=E8=B7=A8?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=20Wiki=20=E6=90=9C=E7=B4=A2=20POST?= =?UTF-8?q?=20/wiki-search=20(#3535)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(wiki): add cross-KB wiki search POST /wiki-search Give external callers a knowledge-search counterpart for wiki pages: one SQL IN query ranked by title/slug/summary/content, slim hits with match_snippet, and Viewer retrieve auth including shared KBs. * fix(wiki): satisfy golangci-lint on cross-KB wiki search Check c.Error returns, wrap lines over 120 columns, and use wikiDialect instead of the embedded Dialector selector. * fix(wiki): address review on cross-KB search helpers Share KB viewer auth and snippet extraction, keep ID limits in the service layer, and escape SQLite LIKE wildcards. * fix(lint): wrap knowledge span fixtures over 120 columns Go Lint still diffs from the original PR base, so later main tests in knowledge_spans_test.go were reported as new line-length issues. * fix(wiki): authorize KB access before checking wiki enablement in cross-KB search Checking IsWikiEnabled before AuthorizeKBAccess let a caller tell apart a foreign KB with wiki disabled (400) from one it cannot see (404), leaking the KB's existence and settings. Run the access check first. --------- Co-authored-by: root Co-authored-by: wizardchen Co-authored-by: lyingbug --- internal/agent/tools/wiki_tools.go | 44 +--- internal/application/access/kb_search.go | 65 ++++++ internal/application/repository/wiki_page.go | 84 ++++++-- .../application/repository/wiki_page_test.go | 104 ++++++++++ .../knowledgebase_search_storegroup.go | 52 +---- internal/application/service/wiki_page.go | 110 ++++++++++ .../service/wiki_page_revision_test.go | 2 +- .../application/service/wiki_page_test.go | 10 +- .../application/service/wiki_search_test.go | 194 ++++++++++++++++++ internal/handler/knowledge_spans_test.go | 49 ++++- internal/handler/wiki_page.go | 113 ++++++++++ internal/handler/wiki_search_test.go | 191 +++++++++++++++++ .../router_api_key_capabilities_test.go | 1 + internal/router/routes_knowledge.go | 5 + internal/searchutil/snippet.go | 50 +++++ internal/searchutil/snippet_test.go | 26 +++ internal/types/interfaces/wiki_page.go | 8 + website-docs/04-api/01-api-overview.md | 2 +- website-docs/04-api/02-api-faq-wiki.md | 41 +++- 19 files changed, 1030 insertions(+), 121 deletions(-) create mode 100644 internal/application/access/kb_search.go create mode 100644 internal/application/service/wiki_search_test.go create mode 100644 internal/handler/wiki_search_test.go create mode 100644 internal/searchutil/snippet.go create mode 100644 internal/searchutil/snippet_test.go diff --git a/internal/agent/tools/wiki_tools.go b/internal/agent/tools/wiki_tools.go index be1784fd..fd55d695 100644 --- a/internal/agent/tools/wiki_tools.go +++ b/internal/agent/tools/wiki_tools.go @@ -12,6 +12,7 @@ import ( "unicode/utf8" "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -955,7 +956,7 @@ func (t *wikiSearchTool) Execute(ctx context.Context, args json.RawMessage) (*ty t.seenSlugs[key] = true t.mu.Unlock() - snippet := extractSnippet(p.Content, pattern) + snippet := searchutil.ExtractSnippet(p.Content, pattern) snippetTag := "" if snippet != "" { snippetTag = fmt.Sprintf("\n%s", snippet) @@ -1050,47 +1051,6 @@ func parseStringOrArray(val any) []string { return nil } -func extractSnippet(content string, query string) string { - if content == "" || query == "" { - return "" - } - re, err := regexp.Compile("(?i)" + query) - if err != nil { - return "" - } - loc := re.FindStringIndex(content) - if loc == nil { - return "" - } - - matchStr := content[loc[0]:loc[1]] - before := content[:loc[0]] - after := content[loc[1]:] - - beforeRunes := []rune(before) - if len(beforeRunes) > 60 { - beforeRunes = beforeRunes[len(beforeRunes)-60:] - } - - afterRunes := []rune(after) - if len(afterRunes) > 60 { - afterRunes = afterRunes[:60] - } - - matchRunes := []rune(matchStr) - if len(matchRunes) > 100 { - matchRunes = append(matchRunes[:100], []rune("...")...) - } - - snippet := string(beforeRunes) + string(matchRunes) + string(afterRunes) - snippet = strings.ReplaceAll(snippet, "\n", " ") - for strings.Contains(snippet, " ") { - snippet = strings.ReplaceAll(snippet, " ", " ") - } - - return "... " + strings.TrimSpace(snippet) + " ..." -} - func truncateRunes(s string, maxRunes int) string { runes := []rune(s) if len(runes) <= maxRunes { diff --git a/internal/application/access/kb_search.go b/internal/application/access/kb_search.go new file mode 100644 index 00000000..ce29cc4a --- /dev/null +++ b/internal/application/access/kb_search.go @@ -0,0 +1,65 @@ +package access + +import ( + "context" + + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// AuthorizeKBAccess rejects multi-KB searches whose scope includes a KB +// that the original caller is not entitled to read. Exact upstream grants +// support shared KB/agent execution; other foreign KBs need an organization +// permission check for the caller via KBPermissions, applying the 3-D cap +// (share role + caller's tenant-org role + tenant Viewer cap) introduced +// in Plan 3 of #1303. +// +// Returning NotFound rather than Forbidden avoids leaking the existence +// of unauthorized KB IDs that the caller could not otherwise observe. +// Structured logs record the rejection with the offending kb_id (always +// safe — KB IDs are UUIDs without sensitive content) and the requesting +// tenant for audit. +func AuthorizeKBAccess( + ctx context.Context, + shares KBShareLookup, + kbs []*types.KnowledgeBase, +) error { + if len(kbs) == 0 { + return nil + } + + kbIDs := make([]string, 0, len(kbs)) + for _, kb := range kbs { + kbIDs = append(kbIDs, kb.ID) + } + if err := types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbIDs...); err != nil { + return err + } + + requestTenantID := types.CallerFromContext(ctx).TenantID + permissions := NewKBPermissions(ctx, shares) + + for _, kb := range kbs { + hasPermission, permErr := permissions.Check(kb.ID, kb.TenantID, types.OrgRoleViewer) + if permErr != nil { + logger.ErrorWithFields(ctx, permErr, map[string]interface{}{ + "caller_tenant_id": requestTenantID, + "kb_tenant_id": kb.TenantID, + "kb_id": kb.ID, + "reason": "shared-KB permission lookup failed", + }) + return apperrors.NewInternalServerError("failed to verify knowledge base access") + } + if !hasPermission { + logger.WarnWithFields(ctx, logger.Fields{ + "caller_tenant_id": requestTenantID, + "kb_tenant_id": kb.TenantID, + "kb_id": kb.ID, + "reason": "tenant lacks viewer permission for foreign-tenant KB", + }, "search scope rejected: unauthorized foreign-tenant KB") + return apperrors.NewNotFoundError("knowledge base not found") + } + } + return nil +} diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index a868d933..28430a72 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -1315,8 +1315,9 @@ func escapeLikePattern(s string) string { return replacer.Replace(s) } -// Search performs case-insensitive POSIX regex search on wiki pages within a knowledge base. -// The query is interpreted as a PostgreSQL regular expression (via ~*). +// Search performs full-text search on wiki pages within a knowledge base. +// PostgreSQL interprets the query as a case-insensitive POSIX regex (~*); +// SQLite / Lite matches a literal substring with LIKE (wildcards escaped). // // Results are ranked by where the query hit, highest-relevance first: // @@ -1330,7 +1331,53 @@ func escapeLikePattern(s string) string { // see pages like "华为" or "Index" ahead of the actual 王新 page just // because they mention 王新 in their body and were updated more recently. // updated_at stays as the tiebreaker so same-rank ties stay deterministic. -func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) { +func (r *wikiPageRepository) Search( + ctx context.Context, kbID string, query string, limit int, +) ([]*types.WikiPage, error) { + return r.SearchAcross(ctx, []string{kbID}, query, limit) +} + +func wikiSearchPredicate(column, op string, withEscape bool) string { + if withEscape { + return column + " " + op + " ? ESCAPE ?" + } + return column + " " + op + " ?" +} + +func wikiSearchMatchArgs(pattern, escape string, withEscape bool, copies int) []interface{} { + n := copies + if withEscape { + n *= 2 + } + args := make([]interface{}, 0, n) + for i := 0; i < copies; i++ { + args = append(args, pattern) + if withEscape { + args = append(args, escape) + } + } + return args +} + +func (r *wikiPageRepository) wikiSearchMatchOp() (op string, withEscape bool) { + // SQLite / Lite (desktop single-binary) has no POSIX ~*; LIKE matches + // a literal substring. Postgres production keeps ~*. + if r.wikiDialect() == "sqlite" { + return "LIKE", true + } + return "~*", false +} + +// SearchAcross searches wiki pages in the given knowledge bases with the +// same field-priority rank as the single-KB Search path, then truncates +// to a global top-N. SQLite / Lite uses LIKE with an explicit ESCAPE; +// Postgres uses POSIX ~*. +func (r *wikiPageRepository) SearchAcross( + ctx context.Context, kbIDs []string, query string, limit int, +) ([]*types.WikiPage, error) { + if len(kbIDs) == 0 { + return nil, nil + } if limit <= 0 { limit = 10 } @@ -1338,22 +1385,31 @@ func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query stri limit = 50 } - // CASE expression is evaluated per-row during SELECT; we order by the - // alias so the DB only computes the rank once. Parameterized four - // times with the same regex to avoid coupling to GORM's positional - // arg rewriting quirks. + op, withEscape := r.wikiSearchMatchOp() + pattern := query + if withEscape { + pattern = "%" + escapeLikePattern(query) + "%" + } + pred := func(column string) string { + return wikiSearchPredicate(column, op, withEscape) + } rankExpr := "CASE " + - "WHEN title ~* ? THEN 4 " + - "WHEN slug ~* ? THEN 3 " + - "WHEN summary ~* ? THEN 2 " + - "WHEN content ~* ? THEN 1 " + + "WHEN " + pred("title") + " THEN 4 " + + "WHEN " + pred("slug") + " THEN 3 " + + "WHEN " + pred("summary") + " THEN 2 " + + "WHEN " + pred("content") + " THEN 1 " + "ELSE 0 END AS match_rank" + matchClause := "(" + pred("title") + " OR " + pred("content") + + " OR " + pred("summary") + " OR " + pred("slug") + ")" + matchArgs := wikiSearchMatchArgs(pattern, likeEscapeChar, withEscape, 4) + whereArgs := make([]interface{}, 0, 1+len(matchArgs)) + whereArgs = append(whereArgs, kbIDs) + whereArgs = append(whereArgs, matchArgs...) var pages []*types.WikiPage if err := r.db.WithContext(ctx). - Select("*, "+rankExpr, query, query, query, query). - Where("knowledge_base_id = ? AND (title ~* ? OR content ~* ? OR summary ~* ? OR slug ~* ?)", - kbID, query, query, query, query). + Select("*, "+rankExpr, matchArgs...). + Where("knowledge_base_id IN ? AND "+matchClause, whereArgs...). Where("status != ?", "archived"). Order("match_rank DESC, updated_at DESC"). Limit(limit). diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go index c2651ed9..bda1c078 100644 --- a/internal/application/repository/wiki_page_test.go +++ b/internal/application/repository/wiki_page_test.go @@ -783,3 +783,107 @@ func TestUpdateIssueStatus_ScopedToKnowledgeBase(t *testing.T) { // databases count matched rows, and updated_at changes anyway. require.NoError(t, repo.UpdateIssueStatus(context.Background(), "kb-a", "issue-1", "resolved")) } + +func TestSearchAcross_RanksTitleAboveContentAndStaysInRequestedKBs(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + titleHit := makeWikiPage("kb-b", "entity/wangxin", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + titleHit.Title = "王新" + titleHit.Content = "unrelated body" + titleHit.UpdatedAt = time.Now().Add(-time.Hour) + + contentHit := makeWikiPage("kb-a", "entity/huawei", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + contentHit.Title = "华为" + contentHit.Content = "mentions 王新 in passing" + contentHit.UpdatedAt = time.Now() + + archived := makeWikiPage("kb-a", "entity/wangxin-old", types.WikiPageTypeEntity, types.WikiPageStatusArchived) + archived.Title = "王新" + + outside := makeWikiPage("kb-c", "entity/wangxin-other", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + outside.Title = "王新" + + for _, p := range []*types.WikiPage{titleHit, contentHit, archived, outside} { + require.NoError(t, repo.Create(ctx, p)) + } + + pages, err := repo.SearchAcross(ctx, []string{"kb-a", "kb-b"}, "王新", 10) + require.NoError(t, err) + require.Len(t, pages, 2) + assert.Equal(t, titleHit.ID, pages[0].ID, "title match in another KB must outrank a newer body mention") + assert.Equal(t, contentHit.ID, pages[1].ID) +} + +func TestSearch_UsesSharedAcrossQueryForSingleKB(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + hit := makeWikiPage("kb-a", "concept/deploy", types.WikiPageTypeConcept, types.WikiPageStatusPublished) + hit.Title = "部署" + miss := makeWikiPage("kb-b", "concept/deploy-other", types.WikiPageTypeConcept, types.WikiPageStatusPublished) + miss.Title = "部署" + require.NoError(t, repo.Create(ctx, hit)) + require.NoError(t, repo.Create(ctx, miss)) + + pages, err := repo.Search(ctx, "kb-a", "部署", 10) + require.NoError(t, err) + require.Len(t, pages, 1) + assert.Equal(t, hit.ID, pages[0].ID) +} + +func TestSearchAcross_ClampsLimit(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + for i := 0; i < 3; i++ { + p := makeWikiPage("kb-a", "entity/p"+strconv.Itoa(i), types.WikiPageTypeEntity, types.WikiPageStatusPublished) + p.Title = "match-term" + p.UpdatedAt = time.Now().Add(time.Duration(i) * time.Minute) + require.NoError(t, repo.Create(ctx, p)) + } + + pages, err := repo.SearchAcross(ctx, []string{"kb-a"}, "match-term", 2) + require.NoError(t, err) + require.Len(t, pages, 2) + + empty, err := repo.SearchAcross(ctx, nil, "match-term", 10) + require.NoError(t, err) + assert.Empty(t, empty) +} + +func TestSearchAcross_SQLiteLikeEscapesWildcards(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + literalPct := makeWikiPage("kb-a", "entity/pct", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + literalPct.Title = "100% done" + other := makeWikiPage("kb-a", "entity/other", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + other.Title = "other page" + literalUnderscore := makeWikiPage("kb-a", "entity/under", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + literalUnderscore.Title = "a_b" + plain := makeWikiPage("kb-a", "entity/plain", types.WikiPageTypeEntity, types.WikiPageStatusPublished) + plain.Title = "axb" + for _, p := range []*types.WikiPage{literalPct, other, literalUnderscore, plain} { + require.NoError(t, repo.Create(ctx, p)) + } + + pages, err := repo.SearchAcross(ctx, []string{"kb-a"}, "%", 10) + require.NoError(t, err) + require.Len(t, pages, 1) + assert.Equal(t, literalPct.ID, pages[0].ID) + + pages, err = repo.SearchAcross(ctx, []string{"kb-a"}, "100%", 10) + require.NoError(t, err) + require.Len(t, pages, 1) + assert.Equal(t, literalPct.ID, pages[0].ID) + + pages, err = repo.SearchAcross(ctx, []string{"kb-a"}, "a_b", 10) + require.NoError(t, err) + require.Len(t, pages, 1) + assert.Equal(t, literalUnderscore.ID, pages[0].ID) +} diff --git a/internal/application/service/knowledgebase_search_storegroup.go b/internal/application/service/knowledgebase_search_storegroup.go index 7f8c724b..00f5dc69 100644 --- a/internal/application/service/knowledgebase_search_storegroup.go +++ b/internal/application/service/knowledgebase_search_storegroup.go @@ -184,59 +184,13 @@ func classifyFactoryError( } } -// authorizeKBAccess rejects multi-KB searches whose scope includes a KB -// that the original caller is not entitled to read. Exact upstream grants -// support shared KB/agent execution; other foreign KBs need an organization -// permission check for the caller via access.KBPermissions, -// applying the 3-D cap (share role + caller's tenant-org role + tenant -// Viewer cap) introduced in Plan 3 of #1303. -// -// Returning NotFound rather than Forbidden avoids leaking the existence -// of unauthorized KB IDs that the caller could not otherwise observe. -// Structured logs record the rejection with the offending kb_id (always -// safe — KB IDs are UUIDs without sensitive content) and the requesting -// tenant for audit. +// authorizeKBAccess is a service wrapper around access.AuthorizeKBAccess +// so HybridSearch and its tests keep calling through knowledgeBaseService. func (s *knowledgeBaseService) authorizeKBAccess( ctx context.Context, kbs []*types.KnowledgeBase, ) error { - if len(kbs) == 0 { - return nil - } - - kbIDs := make([]string, 0, len(kbs)) - for _, kb := range kbs { - kbIDs = append(kbIDs, kb.ID) - } - if err := types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbIDs...); err != nil { - return err - } - - requestTenantID := types.CallerFromContext(ctx).TenantID - permissions := access.NewKBPermissions(ctx, s.kbShareService) - - for _, kb := range kbs { - hasPermission, permErr := permissions.Check(kb.ID, kb.TenantID, types.OrgRoleViewer) - if permErr != nil { - logger.ErrorWithFields(ctx, permErr, map[string]interface{}{ - "caller_tenant_id": requestTenantID, - "kb_tenant_id": kb.TenantID, - "kb_id": kb.ID, - "reason": "shared-KB permission lookup failed", - }) - return apperrors.NewInternalServerError("failed to verify knowledge base access") - } - if !hasPermission { - logger.WarnWithFields(ctx, logger.Fields{ - "caller_tenant_id": requestTenantID, - "kb_tenant_id": kb.TenantID, - "kb_id": kb.ID, - "reason": "tenant lacks viewer permission for foreign-tenant KB", - }, "search scope rejected: unauthorized foreign-tenant KB") - return apperrors.NewNotFoundError("knowledge base not found") - } - } - return nil + return access.AuthorizeKBAccess(ctx, s.kbShareService, kbs) } // validateSameEmbeddingModel rejects multi-KB searches that span more than diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index 83fb0990..6605ab46 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -11,7 +11,9 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/repository" + apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -45,10 +47,13 @@ type wikiPageService struct { repo interfaces.WikiPageRepository chunkRepo interfaces.ChunkRepository kbService interfaces.KnowledgeBaseService + kbShareService interfaces.KBShareService taskPendingRepo interfaces.TaskPendingOpsRepository redisClient *redis.Client } +const maxWikiSearchKnowledgeBases = 32 + // NewWikiPageService creates a new wiki page service func NewWikiPageService( repo interfaces.WikiPageRepository, @@ -56,11 +61,13 @@ func NewWikiPageService( kbService interfaces.KnowledgeBaseService, taskPendingRepo interfaces.TaskPendingOpsRepository, redisClient *redis.Client, + kbShareService interfaces.KBShareService, ) interfaces.WikiPageService { return &wikiPageService{ repo: repo, chunkRepo: chunkRepo, kbService: kbService, + kbShareService: kbShareService, taskPendingRepo: taskPendingRepo, redisClient: redisClient, } @@ -1022,6 +1029,109 @@ func (s *wikiPageService) SearchPages(ctx context.Context, kbID string, query st return s.repo.Search(ctx, kbID, query, limit) } +func (s *wikiPageService) SearchPagesAcross( + ctx context.Context, kbIDs []string, query string, limit int, +) ([]*types.WikiPage, error) { + query = strings.TrimSpace(query) + if query == "" { + return nil, apperrors.NewBadRequestError("query is required") + } + + ids := uniqueNonEmptyWikiSearchKBIDs(kbIDs) + if len(ids) == 0 { + return nil, apperrors.NewBadRequestError( + "at least one knowledge_base_id or knowledge_base_ids must be provided") + } + if len(ids) > maxWikiSearchKnowledgeBases { + return nil, apperrors.NewBadRequestError( + fmt.Sprintf("at most %d knowledge_base_ids are allowed", maxWikiSearchKnowledgeBases)) + } + + if s.kbService == nil { + return nil, apperrors.NewInternalServerError("knowledge base service is not configured") + } + + kbs, err := s.kbService.GetKnowledgeBasesByIDsOnly(ctx, ids) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_ids": ids, + }) + return nil, err + } + + kbByID := make(map[string]*types.KnowledgeBase, len(kbs)) + for _, kb := range kbs { + if kb == nil || kb.ID == "" { + continue + } + kbByID[kb.ID] = kb + } + + authorized := make([]*types.KnowledgeBase, 0, len(ids)) + for _, id := range ids { + kb, ok := kbByID[id] + if !ok { + return nil, apperrors.NewNotFoundError("knowledge base not found") + } + authorized = append(authorized, kb) + } + + if types.CallerFromContext(ctx).TenantID == 0 { + return nil, apperrors.NewUnauthorizedError("tenant id is required") + } + // Authorize before inspecting KB settings so an unauthorized caller gets + // the same NotFound for a foreign KB regardless of its wiki setting. + if err := access.AuthorizeKBAccess(ctx, s.kbShareService, authorized); err != nil { + return nil, err + } + for _, kb := range authorized { + if !kb.IsWikiEnabled() { + return nil, apperrors.NewBadRequestError("Wiki feature is not enabled for this knowledge base") + } + } + + pages, err := s.repo.SearchAcross(ctx, ids, query, limit) + if err != nil { + if isInvalidWikiSearchQuery(err) { + return nil, apperrors.NewBadRequestError("invalid search query") + } + return nil, err + } + if pages == nil { + return []*types.WikiPage{}, nil + } + return pages, nil +} + +func uniqueNonEmptyWikiSearchKBIDs(ids []string) []string { + if len(ids) == 0 { + return nil + } + seen := make(map[string]struct{}, len(ids)) + out := make([]string, 0, len(ids)) + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + return out +} + +func isInvalidWikiSearchQuery(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "invalid regular expression") || + strings.Contains(msg, "invalid regex") +} + // --- Internal helpers --- // parseOutLinks extracts [[wiki-link]] slugs from markdown content diff --git a/internal/application/service/wiki_page_revision_test.go b/internal/application/service/wiki_page_revision_test.go index f351bb03..8d3a016b 100644 --- a/internal/application/service/wiki_page_revision_test.go +++ b/internal/application/service/wiki_page_revision_test.go @@ -23,7 +23,7 @@ func newWikiRevisionTestService(t *testing.T) (context.Context, wikiRevisionTest require.NoError(t, err) require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{}, &types.WikiPageRevision{})) repo := repository.NewWikiPageRepository(db) - svc := NewWikiPageService(repo, nil, nil, nil, nil) + svc := NewWikiPageService(repo, nil, nil, nil, nil, nil) return context.Background(), wikiRevisionTestHarness{svc: svc}, db } diff --git a/internal/application/service/wiki_page_test.go b/internal/application/service/wiki_page_test.go index 1c315f25..dd4a5521 100644 --- a/internal/application/service/wiki_page_test.go +++ b/internal/application/service/wiki_page_test.go @@ -21,7 +21,7 @@ func TestPruneEmptyFolderChainsDeletesOnlyEmptyCandidateAncestors(t *testing.T) ctx := context.Background() repo := repository.NewWikiPageRepository(db) - svc := NewWikiPageService(repo, nil, nil, nil, nil) + svc := NewWikiPageService(repo, nil, nil, nil, nil, nil) now := time.Now() createFolder := func(id, parentID, name, path string, depth int) { require.NoError(t, repo.CreateFolder(ctx, &types.WikiFolder{ @@ -77,7 +77,7 @@ func TestUpdateWikiPagePersistsAndClearsAliases(t *testing.T) { ctx := context.Background() repo := repository.NewWikiPageRepository(db) - svc := NewWikiPageService(repo, nil, nil, nil, nil) + svc := NewWikiPageService(repo, nil, nil, nil, nil, nil) page, err := svc.CreatePage(ctx, &types.WikiPage{ TenantID: 1, KnowledgeBaseID: "kb-alias", Slug: "concept/alias", Title: "Alias", Summary: "summary", Content: "content", @@ -180,7 +180,7 @@ func TestRepairContentLinks(t *testing.T) { ctx := context.Background() repo := repository.NewWikiPageRepository(db) - svc := NewWikiPageService(repo, nil, nil, nil, nil) + svc := NewWikiPageService(repo, nil, nil, nil, nil, nil) const kbID = "kb-repair" now := time.Now() @@ -613,7 +613,7 @@ func TestFindPagesByNormalizedTitleMatchesWhitespace(t *testing.T) { ctx := context.Background() repo := repository.NewWikiPageRepository(db) - svc := NewWikiPageService(repo, nil, nil, nil, nil) + svc := NewWikiPageService(repo, nil, nil, nil, nil, nil) now := time.Now() require.NoError(t, repo.Create(ctx, &types.WikiPage{ ID: "page-kong", TenantID: 1, KnowledgeBaseID: "kb-id", Slug: "entity/confucius", @@ -671,7 +671,7 @@ func TestMovePageIntoTypeLabelNamedFolderKeepsHierarchy(t *testing.T) { ctx := context.Background() repo := repository.NewWikiPageRepository(db) - svc := NewWikiPageService(repo, nil, nil, nil, nil) + svc := NewWikiPageService(repo, nil, nil, nil, nil, nil) now := time.Now() folder, err := svc.CreateFolder(ctx, "kb-move", 1, types.WikiFolderRootID, "概念") diff --git a/internal/application/service/wiki_search_test.go b/internal/application/service/wiki_search_test.go new file mode 100644 index 00000000..83b0851a --- /dev/null +++ b/internal/application/service/wiki_search_test.go @@ -0,0 +1,194 @@ +package service + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/application/repository" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type wikiSearchKBService struct { + interfaces.KnowledgeBaseService + byID map[string]*types.KnowledgeBase +} + +func (s *wikiSearchKBService) GetKnowledgeBasesByIDsOnly( + _ context.Context, ids []string, +) ([]*types.KnowledgeBase, error) { + out := make([]*types.KnowledgeBase, 0, len(ids)) + for _, id := range ids { + if kb, ok := s.byID[id]; ok { + out = append(out, kb) + } + } + return out, nil +} + +type wikiSearchShareService struct { + interfaces.KBShareService + allowed map[string]map[uint64]bool + err error +} + +func (s *wikiSearchShareService) CheckTenantKBPermission( + _ context.Context, kbID string, callerTenantID uint64, _ types.TenantRole, +) (types.OrgMemberRole, bool, error) { + if s.err != nil { + return "", false, s.err + } + return types.OrgRoleViewer, s.allowed[kbID][callerTenantID], nil +} + +func makeWikiSearchKB(id string, tenantID uint64) *types.KnowledgeBase { + return &types.KnowledgeBase{ + ID: id, + TenantID: tenantID, + IndexingStrategy: types.IndexingStrategy{WikiEnabled: true}, + } +} + +func setupWikiSearchService( + t *testing.T, + kbs map[string]*types.KnowledgeBase, + share interfaces.KBShareService, +) (context.Context, interfaces.WikiPageService, interfaces.WikiPageRepository) { + t.Helper() + db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{}, &types.WikiPageRevision{})) + repo := repository.NewWikiPageRepository(db) + svc := NewWikiPageService(repo, nil, &wikiSearchKBService{byID: kbs}, nil, nil, share) + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + return ctx, svc, repo +} + +func TestSearchPagesAcross_RanksAcrossEnabledKBs(t *testing.T) { + ctx, svc, repo := setupWikiSearchService(t, map[string]*types.KnowledgeBase{ + "kb-a": makeWikiSearchKB("kb-a", 1), + "kb-b": makeWikiSearchKB("kb-b", 1), + }, nil) + + titleHit := &types.WikiPage{ + ID: "p-title", TenantID: 1, KnowledgeBaseID: "kb-b", + Slug: "entity/wangxin", Title: "王新", PageType: types.WikiPageTypeEntity, + Status: types.WikiPageStatusPublished, Content: "other", Version: 1, + CreatedAt: time.Now(), UpdatedAt: time.Now().Add(-time.Hour), + } + contentHit := &types.WikiPage{ + ID: "p-body", TenantID: 1, KnowledgeBaseID: "kb-a", + Slug: "entity/huawei", Title: "华为", PageType: types.WikiPageTypeEntity, + Status: types.WikiPageStatusPublished, Content: "mentions 王新", Version: 1, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + require.NoError(t, repo.Create(ctx, titleHit)) + require.NoError(t, repo.Create(ctx, contentHit)) + + pages, err := svc.SearchPagesAcross(ctx, []string{"kb-a", "kb-b"}, "王新", 10) + require.NoError(t, err) + require.Len(t, pages, 2) + assert.Equal(t, "p-title", pages[0].ID) + assert.Equal(t, "p-body", pages[1].ID) +} + +func TestSearchPagesAcross_RejectsNonWikiKB(t *testing.T) { + ctx, svc, _ := setupWikiSearchService(t, map[string]*types.KnowledgeBase{ + "kb-wiki": makeWikiSearchKB("kb-wiki", 1), + "kb-doc": { + ID: "kb-doc", TenantID: 1, + IndexingStrategy: types.IndexingStrategy{VectorEnabled: true}, + }, + }, nil) + + _, err := svc.SearchPagesAcross(ctx, []string{"kb-wiki", "kb-doc"}, "q", 10) + require.Error(t, err) + app, ok := apperrors.IsAppError(err) + require.True(t, ok) + assert.Equal(t, apperrors.ErrBadRequest, app.Code) +} + +func TestSearchPagesAcross_MissingKBIsNotFound(t *testing.T) { + ctx, svc, _ := setupWikiSearchService(t, map[string]*types.KnowledgeBase{ + "kb-wiki": makeWikiSearchKB("kb-wiki", 1), + }, nil) + + _, err := svc.SearchPagesAcross(ctx, []string{"kb-wiki", "kb-missing"}, "q", 10) + require.Error(t, err) + app, ok := apperrors.IsAppError(err) + require.True(t, ok) + assert.Equal(t, apperrors.ErrNotFound, app.Code) +} + +func TestSearchPagesAcross_ForeignTenantWithoutShareIsNotFound(t *testing.T) { + ctx, svc, _ := setupWikiSearchService(t, map[string]*types.KnowledgeBase{ + "kb-foreign": makeWikiSearchKB("kb-foreign", 99), + }, &wikiSearchShareService{}) + + _, err := svc.SearchPagesAcross(ctx, []string{"kb-foreign"}, "q", 10) + require.Error(t, err) + app, ok := apperrors.IsAppError(err) + require.True(t, ok) + assert.Equal(t, apperrors.ErrNotFound, app.Code) +} + +func TestSearchPagesAcross_ForeignTenantNonWikiWithoutShareIsNotFound(t *testing.T) { + ctx, svc, _ := setupWikiSearchService(t, map[string]*types.KnowledgeBase{ + "kb-foreign-doc": { + ID: "kb-foreign-doc", TenantID: 99, + IndexingStrategy: types.IndexingStrategy{VectorEnabled: true}, + }, + }, &wikiSearchShareService{}) + + _, err := svc.SearchPagesAcross(ctx, []string{"kb-foreign-doc"}, "q", 10) + require.Error(t, err) + app, ok := apperrors.IsAppError(err) + require.True(t, ok) + assert.Equal(t, apperrors.ErrNotFound, app.Code) +} + +func TestSearchPagesAcross_ForeignTenantWithShareOK(t *testing.T) { + ctx, svc, repo := setupWikiSearchService(t, map[string]*types.KnowledgeBase{ + "kb-foreign": makeWikiSearchKB("kb-foreign", 99), + }, &wikiSearchShareService{ + allowed: map[string]map[uint64]bool{"kb-foreign": {1: true}}, + }) + + page := &types.WikiPage{ + ID: "p-shared", TenantID: 99, KnowledgeBaseID: "kb-foreign", + Slug: "concept/share", Title: "共享", PageType: types.WikiPageTypeConcept, + Status: types.WikiPageStatusPublished, Content: "body", Version: 1, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + require.NoError(t, repo.Create(ctx, page)) + + pages, err := svc.SearchPagesAcross(ctx, []string{"kb-foreign"}, "共享", 10) + require.NoError(t, err) + require.Len(t, pages, 1) + assert.Equal(t, "p-shared", pages[0].ID) +} + +func TestSearchPagesAcross_RejectsTooManyKBs(t *testing.T) { + ids := make([]string, maxWikiSearchKnowledgeBases+1) + for i := range ids { + ids[i] = fmt.Sprintf("kb-%d", i) + } + ctx, svc, _ := setupWikiSearchService(t, nil, &wikiSearchShareService{}) + _, err := svc.SearchPagesAcross(ctx, ids, "q", 10) + require.Error(t, err) + app, ok := apperrors.IsAppError(err) + require.True(t, ok) + assert.Equal(t, apperrors.ErrBadRequest, app.Code) +} + +func TestIsInvalidWikiSearchQuery(t *testing.T) { + assert.True(t, isInvalidWikiSearchQuery(fmt.Errorf("pq: invalid regular expression: quantifier operand invalid"))) + assert.False(t, isInvalidWikiSearchQuery(fmt.Errorf("connection refused"))) +} diff --git a/internal/handler/knowledge_spans_test.go b/internal/handler/knowledge_spans_test.go index f7ec4958..2b882a03 100644 --- a/internal/handler/knowledge_spans_test.go +++ b/internal/handler/knowledge_spans_test.go @@ -190,9 +190,22 @@ func TestBuildSpanTree_MissingStagesAroundRealFailure(t *testing.T) { now := time.Now() finished := now.Add(2 * time.Second) rows := []types.KnowledgeProcessingSpan{ - {KnowledgeID: "kid", Attempt: 1, SpanID: "root", Name: "knowledge_processing", Kind: types.SpanKindRoot, Status: types.SpanStatusFailed, StartedAt: &now, FinishedAt: &finished}, - {KnowledgeID: "kid", Attempt: 1, SpanID: "chunk", ParentSpanID: "root", Name: types.StageChunking, Kind: types.SpanKindStage, Status: types.SpanStatusDone, StartedAt: &now, FinishedAt: &finished}, - {KnowledgeID: "kid", Attempt: 1, SpanID: "emb", ParentSpanID: "root", Name: types.StageEmbedding, Kind: types.SpanKindStage, Status: types.SpanStatusFailed, ErrorCode: "EMBED_UNREACHABLE", StartedAt: &now, FinishedAt: &finished}, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "root", + Name: "knowledge_processing", Kind: types.SpanKindRoot, + Status: types.SpanStatusFailed, StartedAt: &now, FinishedAt: &finished, + }, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "chunk", ParentSpanID: "root", + Name: types.StageChunking, Kind: types.SpanKindStage, + Status: types.SpanStatusDone, StartedAt: &now, FinishedAt: &finished, + }, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "emb", ParentSpanID: "root", + Name: types.StageEmbedding, Kind: types.SpanKindStage, + Status: types.SpanStatusFailed, ErrorCode: "EMBED_UNREACHABLE", + StartedAt: &now, FinishedAt: &finished, + }, } tree, _, lastFail := buildSpanTree("kid", 1, rows, types.ParseStatusFailed) @@ -226,8 +239,16 @@ func TestBuildSpanTree_MissingStagesAroundRealFailure(t *testing.T) { func TestBuildSpanTree_MissingStagesWithoutFailureKeepFallback(t *testing.T) { now := time.Now() rows := []types.KnowledgeProcessingSpan{ - {KnowledgeID: "kid", Attempt: 1, SpanID: "root", Name: "knowledge_processing", Kind: types.SpanKindRoot, Status: types.SpanStatusRunning, StartedAt: &now}, - {KnowledgeID: "kid", Attempt: 1, SpanID: "doc", ParentSpanID: "root", Name: types.StageDocReader, Kind: types.SpanKindStage, Status: types.SpanStatusDone, StartedAt: &now}, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "root", + Name: "knowledge_processing", Kind: types.SpanKindRoot, + Status: types.SpanStatusRunning, StartedAt: &now, + }, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "doc", ParentSpanID: "root", + Name: types.StageDocReader, Kind: types.SpanKindStage, + Status: types.SpanStatusDone, StartedAt: &now, + }, } tree, _, _ := buildSpanTree("kid", 1, rows, types.ParseStatusProcessing) @@ -255,15 +276,27 @@ func TestBuildSpanTree_CancelledParseRendersCancelled(t *testing.T) { func TestBuildSpanTree_DocReaderFailureCancelsEverythingAfter(t *testing.T) { now := time.Now() rows := []types.KnowledgeProcessingSpan{ - {KnowledgeID: "kid", Attempt: 1, SpanID: "root", Name: "knowledge_processing", Kind: types.SpanKindRoot, Status: types.SpanStatusFailed, StartedAt: &now}, - {KnowledgeID: "kid", Attempt: 1, SpanID: "doc", ParentSpanID: "root", Name: types.StageDocReader, Kind: types.SpanKindStage, Status: types.SpanStatusFailed, ErrorCode: "DOCREADER_TIMEOUT", StartedAt: &now}, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "root", + Name: "knowledge_processing", Kind: types.SpanKindRoot, + Status: types.SpanStatusFailed, StartedAt: &now, + }, + { + KnowledgeID: "kid", Attempt: 1, SpanID: "doc", ParentSpanID: "root", + Name: types.StageDocReader, Kind: types.SpanKindStage, + Status: types.SpanStatusFailed, ErrorCode: "DOCREADER_TIMEOUT", + StartedAt: &now, + }, } tree, _, _ := buildSpanTree("kid", 1, rows, types.ParseStatusFailed) got := stageStatuses(t, tree) a := assert.New(t) a.Equal(types.SpanStatusFailed, got[types.StageDocReader]) - for _, name := range []string{types.StageChunking, types.StageEmbedding, types.StageMultimodal, types.StagePostProcess} { + for _, name := range []string{ + types.StageChunking, types.StageEmbedding, + types.StageMultimodal, types.StagePostProcess, + } { a.Equal(types.SpanStatusCancelled, got[name], "%s follows the failed docreader and never ran", name) } diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go index fb44a997..fb8e0700 100644 --- a/internal/handler/wiki_page.go +++ b/internal/handler/wiki_page.go @@ -12,6 +12,7 @@ import ( "github.com/Tencent/WeKnora/internal/application/service" "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -1019,6 +1020,118 @@ func (h *WikiPageHandler) SearchPages(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"pages": pages}) } +// SearchWikiRequest is the body for POST /api/v1/wiki-search. +type SearchWikiRequest struct { + Query string `json:"query" binding:"required"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeBaseIDs []string `json:"knowledge_base_ids"` + Limit int `json:"limit"` +} + +// WikiSearchHit is the slim POST /wiki-search hit. Full page body is on +// GET /knowledgebase/:kb_id/wiki/pages/*slug. +type WikiSearchHit struct { + ID string `json:"id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + Slug string `json:"slug"` + Title string `json:"title"` + PageType string `json:"page_type"` + Aliases types.StringArray `json:"aliases"` + Summary string `json:"summary"` + MatchSnippet string `json:"match_snippet,omitempty"` +} + +// SearchPagesAcross godoc +// @Summary Cross-KB wiki search +// @Description Cross-KB wiki search using the same POSIX regex ranking as single-KB wiki search +// @Tags Wiki +// @Accept json +// @Produce json +// @Param request body SearchWikiRequest true "Wiki search request" +// @Success 200 {object} map[string]interface{} +// @Failure 400 {object} errors.AppError +// @Security Bearer +// @Security ApiKeyAuth +// @Router /wiki-search [post] +func (h *WikiPageHandler) SearchPagesAcross(c *gin.Context) { + ctx := logger.CloneContext(c.Request.Context()) + + var request SearchWikiRequest + if err := c.ShouldBindJSON(&request); err != nil { + _ = c.Error(errors.NewBadRequestError(err.Error())) + return + } + + kbIDs := mergeWikiSearchKBIDs(request.KnowledgeBaseIDs, request.KnowledgeBaseID) + if strings.TrimSpace(request.Query) == "" { + _ = c.Error(errors.NewBadRequestError("query is required")) + return + } + if len(kbIDs) == 0 { + _ = c.Error(errors.NewBadRequestError( + "at least one knowledge_base_id or knowledge_base_ids must be provided")) + return + } + if err := types.AuthorizeTenantAPIKeyKnowledgeTargets(ctx, kbIDs, nil); err != nil { + _ = c.Error(err) + return + } + + logger.Infof(ctx, "Wiki search request, knowledge base IDs: %v, query: %s", + secutils.SanitizeForLogArray(kbIDs), secutils.SanitizeForLog(request.Query)) + + pages, err := h.wikiService.SearchPagesAcross(ctx, kbIDs, request.Query, request.Limit) + if err != nil { + if _, ok := errors.IsAppError(err); ok { + _ = c.Error(err) + return + } + logger.ErrorWithFields(ctx, err, nil) + _ = c.Error(errors.NewInternalServerError(err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": wikiPagesToSearchHits(pages, request.Query), + }) +} + +func wikiPagesToSearchHits(pages []*types.WikiPage, query string) []WikiSearchHit { + hits := make([]WikiSearchHit, 0, len(pages)) + for _, page := range pages { + if page == nil { + continue + } + hits = append(hits, WikiSearchHit{ + ID: page.ID, + KnowledgeBaseID: page.KnowledgeBaseID, + Slug: page.Slug, + Title: page.Title, + PageType: page.PageType, + Aliases: page.Aliases, + Summary: page.Summary, + MatchSnippet: searchutil.ExtractSnippet(page.Content, query), + }) + } + return hits +} + +func mergeWikiSearchKBIDs(ids []string, single string) []string { + out := make([]string, 0, len(ids)+1) + appendID := func(id string) { + id = strings.TrimSpace(id) + if id == "" { + return + } + out = append(out, id) + } + for _, id := range ids { + appendID(id) + } + appendID(single) + return out +} + // RebuildLinks godoc // @Summary Rebuild wiki links // @Description Re-parse all pages and rebuild bidirectional link references diff --git a/internal/handler/wiki_search_test.go b/internal/handler/wiki_search_test.go new file mode 100644 index 00000000..848ea761 --- /dev/null +++ b/internal/handler/wiki_search_test.go @@ -0,0 +1,191 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeWikiSearchService struct { + interfaces.WikiPageService + pages []*types.WikiPage + err error + got struct { + kbIDs []string + query string + limit int + } +} + +func (f *fakeWikiSearchService) SearchPagesAcross( + _ context.Context, kbIDs []string, query string, limit int, +) ([]*types.WikiPage, error) { + f.got.kbIDs = append([]string(nil), kbIDs...) + f.got.query = query + f.got.limit = limit + if f.err != nil { + return nil, f.err + } + return f.pages, nil +} + +func wikiSearchEngine(t *testing.T, h *WikiPageHandler, scope *types.TenantAPIKeyScope) *gin.Engine { + t.Helper() + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(middleware.ErrorHandler()) + r.POST("/wiki-search", func(c *gin.Context) { + ctx := context.WithValue(c.Request.Context(), types.TenantIDContextKey, uint64(1)) + if scope != nil { + ctx = types.WithTenantAPIKeyScope(ctx, *scope) + } + c.Request = c.Request.WithContext(ctx) + c.Set(types.TenantIDContextKey.String(), uint64(1)) + h.SearchPagesAcross(c) + }) + return r +} + +func postWikiSearch(r *gin.Engine, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/wiki-search", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + return w +} + +func TestSearchPagesAcross_ReturnsSuccessData(t *testing.T) { + fake := &fakeWikiSearchService{pages: []*types.WikiPage{ + {ID: "p1", KnowledgeBaseID: "kb-a", Title: "部署"}, + }} + h := &WikiPageHandler{wikiService: fake} + w := postWikiSearch(wikiSearchEngine(t, h, nil), `{"query":"部署","knowledge_base_ids":["kb-a","kb-b"]}`) + require.Equal(t, http.StatusOK, w.Code) + + var got map[string]any + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &got)) + assert.Equal(t, true, got["success"]) + require.NotNil(t, got["data"]) + assert.Equal(t, []string{"kb-a", "kb-b"}, fake.got.kbIDs) + assert.Equal(t, "部署", fake.got.query) +} + +func TestSearchPagesAcross_ReturnsSlimHitsWithoutContent(t *testing.T) { + longBody := strings.Repeat("前", 80) + "部署手册正文" + strings.Repeat("后", 80) + fake := &fakeWikiSearchService{pages: []*types.WikiPage{ + { + ID: "p1", + TenantID: 99, + KnowledgeBaseID: "kb-a", + Slug: "ops/deploy", + Title: "部署", + PageType: "entity", + Status: "published", + Content: longBody, + Summary: "发布步骤", + Aliases: types.StringArray{"发布"}, + Version: 3, + InLinks: types.StringArray{"index"}, + OutLinks: types.StringArray{"ops/rollback"}, + WikiPath: "/ops/deploy", + FolderID: "folder-1", + SourceRefs: types.StringArray{"doc-1|手册"}, + }, + }} + h := &WikiPageHandler{wikiService: fake} + w := postWikiSearch(wikiSearchEngine(t, h, nil), `{"query":"部署","knowledge_base_ids":["kb-a"]}`) + require.Equal(t, http.StatusOK, w.Code) + + var got struct { + Success bool `json:"success"` + Data []map[string]any `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &got)) + require.True(t, got.Success) + require.Len(t, got.Data, 1) + + hit := got.Data[0] + assert.Equal(t, "p1", hit["id"]) + assert.Equal(t, "kb-a", hit["knowledge_base_id"]) + assert.Equal(t, "ops/deploy", hit["slug"]) + assert.Equal(t, "部署", hit["title"]) + assert.Equal(t, "entity", hit["page_type"]) + assert.Equal(t, "发布步骤", hit["summary"]) + assert.Equal(t, []any{"发布"}, hit["aliases"]) + + snippet, ok := hit["match_snippet"].(string) + require.True(t, ok) + assert.Contains(t, snippet, "部署") + assert.NotEqual(t, longBody, snippet) + assert.Less(t, len([]rune(snippet)), 250) + + for _, dropped := range []string{ + "content", "tenant_id", "status", "version", "in_links", "out_links", + "folder_id", "wiki_path", "source_refs", "chunk_refs", "page_metadata", + "created_at", "updated_at", + } { + _, exists := hit[dropped] + assert.False(t, exists, "dropped field %s should be absent", dropped) + } +} + +func TestSearchPagesAcross_MergesSingleKnowledgeBaseID(t *testing.T) { + fake := &fakeWikiSearchService{pages: []*types.WikiPage{}} + h := &WikiPageHandler{wikiService: fake} + body := `{"query":"q","knowledge_base_id":"kb-1","knowledge_base_ids":["kb-2"]}` + w := postWikiSearch(wikiSearchEngine(t, h, nil), body) + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, []string{"kb-2", "kb-1"}, fake.got.kbIDs) +} + +func TestSearchPagesAcross_RequiresQueryAndKBs(t *testing.T) { + h := &WikiPageHandler{wikiService: &fakeWikiSearchService{}} + r := wikiSearchEngine(t, h, nil) + + w := postWikiSearch(r, `{"knowledge_base_ids":["kb-a"]}`) + assert.Equal(t, http.StatusBadRequest, w.Code) + + w = postWikiSearch(r, `{"query":"q"}`) + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestSearchPagesAcross_APIKeyAllowListForbidden(t *testing.T) { + fake := &fakeWikiSearchService{} + h := &WikiPageHandler{wikiService: fake} + scope := &types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"kb-allowed"}} + w := postWikiSearch(wikiSearchEngine(t, h, scope), `{"query":"q","knowledge_base_ids":["kb-other"]}`) + assert.Equal(t, http.StatusForbidden, w.Code) + assert.Nil(t, fake.got.kbIDs) +} + +func TestSearchPagesAcross_PropagatesNotFound(t *testing.T) { + fake := &fakeWikiSearchService{err: errors.NewNotFoundError("knowledge base not found")} + h := &WikiPageHandler{wikiService: fake} + w := postWikiSearch(wikiSearchEngine(t, h, nil), `{"query":"q","knowledge_base_ids":["kb-a"]}`) + assert.Equal(t, http.StatusNotFound, w.Code) +} + +func TestSearchPagesAcross_PropagatesNonWikiBadRequest(t *testing.T) { + fake := &fakeWikiSearchService{ + err: errors.NewBadRequestError("Wiki feature is not enabled for this knowledge base"), + } + h := &WikiPageHandler{wikiService: fake} + w := postWikiSearch(wikiSearchEngine(t, h, nil), `{"query":"q","knowledge_base_ids":["kb-a"]}`) + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestMergeWikiSearchKBIDsMerges(t *testing.T) { + got := mergeWikiSearchKBIDs([]string{" kb-a ", "kb-b", "kb-a"}, "kb-b") + assert.Equal(t, []string{"kb-a", "kb-b", "kb-a", "kb-b"}, got) +} diff --git a/internal/router/router_api_key_capabilities_test.go b/internal/router/router_api_key_capabilities_test.go index 713a0b5d..9c77eb5d 100644 --- a/internal/router/router_api_key_capabilities_test.go +++ b/internal/router/router_api_key_capabilities_test.go @@ -313,6 +313,7 @@ func TestKnowledgeReadRoutesDeclareRetrieveCapability(t *testing.T) { {http.MethodPost, "/api/v1/knowledge-bases/:id/faq/search"}, {http.MethodGet, "/api/v1/knowledge-bases/:id/tags"}, {http.MethodPost, "/api/v1/knowledge-search"}, + {http.MethodPost, "/api/v1/wiki-search"}, {http.MethodGet, "/api/v1/initialization/config/:kbId"}, {http.MethodGet, "/api/v1/knowledgebase/:kb_id/wiki/pages"}, } diff --git a/internal/router/routes_knowledge.go b/internal/router/routes_knowledge.go index c9e9acb6..3e53d505 100644 --- a/internal/router/routes_knowledge.go +++ b/internal/router/routes_knowledge.go @@ -364,4 +364,9 @@ func RegisterWikiPageRoutes(r *gin.RouterGroup, wikiHandler *handler.WikiPageHan wikiRead.GET("/issues", g.Viewer(), g.KBAccessRead("kb_id"), wikiHandler.ListIssues) wiki.PUT("/issues/:issue_id/status", g.OwnedWikiKBOrAdmin(), g.KBAccessWrite("kb_id"), wikiHandler.UpdateIssueStatus) } + + wikiSearch := g.apiKeyGroup(r.Group("/wiki-search", g.Viewer()), apiKeyRetrieve(apiKeyFullAccess())) + { + wikiSearch.POST("", wikiHandler.SearchPagesAcross) + } } diff --git a/internal/searchutil/snippet.go b/internal/searchutil/snippet.go new file mode 100644 index 00000000..bc63309b --- /dev/null +++ b/internal/searchutil/snippet.go @@ -0,0 +1,50 @@ +package searchutil + +import ( + "regexp" + "strings" +) + +// ExtractSnippet matches Agent wiki_search: ~60 runes of context around the +// first case-insensitive regex hit, match itself capped at 100. Invalid +// patterns return an empty string. +func ExtractSnippet(content string, query string) string { + if content == "" || query == "" { + return "" + } + re, err := regexp.Compile("(?i)" + query) + if err != nil { + return "" + } + loc := re.FindStringIndex(content) + if loc == nil { + return "" + } + + matchStr := content[loc[0]:loc[1]] + before := content[:loc[0]] + after := content[loc[1]:] + + beforeRunes := []rune(before) + if len(beforeRunes) > 60 { + beforeRunes = beforeRunes[len(beforeRunes)-60:] + } + + afterRunes := []rune(after) + if len(afterRunes) > 60 { + afterRunes = afterRunes[:60] + } + + matchRunes := []rune(matchStr) + if len(matchRunes) > 100 { + matchRunes = append(matchRunes[:100], []rune("...")...) + } + + snippet := string(beforeRunes) + string(matchRunes) + string(afterRunes) + snippet = strings.ReplaceAll(snippet, "\n", " ") + for strings.Contains(snippet, " ") { + snippet = strings.ReplaceAll(snippet, " ", " ") + } + + return "... " + strings.TrimSpace(snippet) + " ..." +} diff --git a/internal/searchutil/snippet_test.go b/internal/searchutil/snippet_test.go new file mode 100644 index 00000000..d170ef7b --- /dev/null +++ b/internal/searchutil/snippet_test.go @@ -0,0 +1,26 @@ +package searchutil + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtractSnippet_AroundHit(t *testing.T) { + content := strings.Repeat("前", 80) + "部署手册" + strings.Repeat("后", 80) + got := ExtractSnippet(content, "部署") + assert.Contains(t, got, "部署") + assert.True(t, strings.HasPrefix(got, "... ")) + assert.True(t, strings.HasSuffix(got, " ...")) + assert.Less(t, len([]rune(got)), 250) +} + +func TestExtractSnippet_InvalidRegex(t *testing.T) { + assert.Empty(t, ExtractSnippet("hello world", "(")) +} + +func TestExtractSnippet_EmptyInputs(t *testing.T) { + assert.Empty(t, ExtractSnippet("", "q")) + assert.Empty(t, ExtractSnippet("body", "")) +} diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 7db96261..789c341e 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -216,6 +216,10 @@ type WikiPageService interface { // SearchPages performs full-text search over wiki pages. SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) + // SearchPagesAcross searches wiki pages across multiple knowledge bases + // with the same ranking as SearchPages, then returns a global top-N. + SearchPagesAcross(ctx context.Context, kbIDs []string, query string, limit int) ([]*types.WikiPage, error) + // ListRevisions returns the stored historical snapshots for a page // (newest first, content omitted) plus the total snapshot count and the // page's current version. The current version itself has no revision @@ -392,6 +396,10 @@ type WikiPageRepository interface { // Search performs full-text search on wiki pages within a knowledge base. Search(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) + // SearchAcross performs the same full-text search across multiple + // knowledge bases, then returns a global top-N by match_rank. + SearchAcross(ctx context.Context, kbIDs []string, query string, limit int) ([]*types.WikiPage, error) + // CountByType returns page counts grouped by type for a knowledge base. CountByType(ctx context.Context, kbID string) (map[string]int64, error) diff --git a/website-docs/04-api/01-api-overview.md b/website-docs/04-api/01-api-overview.md index 0af0d6fd..7653b3b6 100644 --- a/website-docs/04-api/01-api-overview.md +++ b/website-docs/04-api/01-api-overview.md @@ -351,7 +351,7 @@ rerank 模型加载失败或调用出错时,请求不会失败,而是按召 | 组织与共享 | [02-api-org.md](./02-api-org.md) | `/organizations`、`/shared-*`、`/knowledge-bases/:id/shares`、`/agents/:id/shares` | | 知识库与知识 | [02-api-knowledge.md](./02-api-knowledge.md) | `/knowledge-bases`、`/knowledge`、知识库文件夹 | | 分块与标签 | [02-api-chunks.md](./02-api-chunks.md) | `/chunks`、`/knowledge-bases/:id/tags`、`/chunker/preview` | -| FAQ 与 Wiki | [02-api-faq-wiki.md](./02-api-faq-wiki.md) | `/knowledge-bases/:id/faq`、`/faq`、`/knowledgebase/:kb_id/wiki` | +| FAQ 与 Wiki | [02-api-faq-wiki.md](./02-api-faq-wiki.md) | `/knowledge-bases/:id/faq`、`/faq`、`/knowledgebase/:kb_id/wiki`、`/wiki-search` | | 会话、消息与聊天 | [02-api-chat.md](./02-api-chat.md) | `/sessions`、`/messages`、`/knowledge-chat`、`/agent-chat`、`/knowledge-search` | | 模型与初始化 | [02-api-model-system.md](./02-api-model-system.md) | `/models`、`/initialization`、`/evaluation`、`/weknoracloud` | | 系统与平台管理 | [02-api-system.md](./02-api-system.md) | `/system`、`/system/admin` | diff --git a/website-docs/04-api/02-api-faq-wiki.md b/website-docs/04-api/02-api-faq-wiki.md index bf3cfa58..f959ed4d 100644 --- a/website-docs/04-api/02-api-faq-wiki.md +++ b/website-docs/04-api/02-api-faq-wiki.md @@ -2,7 +2,7 @@ 管理知识库中的 FAQ 条目与 Wiki 页面,支持导入、检索、编辑和版本恢复。 -两组均为 KB 内容子资源:读为 Viewer+ 且 KB read(API key `retrieve`/full);写为“KB 创建者 OR Admin+”且 KB write(API key `ingest`/full),并受 KB 白名单约束。 +两组均为 KB 内容子资源:读为 Viewer+ 且 KB read(API key `retrieve`/full);写为“KB 创建者 OR Admin+”且 KB write(API key `ingest`/full),并受 KB 白名单约束。跨库 Wiki 搜索 `POST /wiki-search` 无路径 KB,Viewer+ 且 API key `retrieve`/full,handler 内校验每个目标库的 allow-list 与共享库 Viewer 权限。 ## FAQ(/api/v1/knowledge-bases/:id/faq) @@ -400,12 +400,51 @@ curl $BASE/api/v1/knowledgebase/kb-1/wiki/stats -H "Authorization: Bearer $TOKEN 用途:页面搜索。查询参数:`q`(必填)、`limit`(默认 10)。 +匹配语义随存储方言变化: + +- Postgres:`q` 按 POSIX 正则(`~*`,大小写不敏感)匹配。 +- SQLite / Lite:按 `LIKE` 字面子串匹配;`%` / `_` 会转义,不当通配符。 + 响应:200 `{"pages":[WikiPage]}` ```bash curl "$BASE/api/v1/knowledgebase/kb-1/wiki/search?q=部署" -H "Authorization: Bearer $TOKEN" ``` +### POST /api/v1/wiki-search + +用途:跨知识库 Wiki 页面搜索(无会话、非混合检索)。Handler: `internal/handler/wiki_page.go` 的 `SearchPagesAcross`。算法与单库 `GET .../wiki/search` 相同(标题 > slug > summary > 正文),对多个库做一次查询后按 `match_rank` 全局截断。API key:`retrieve`/full。 + +| 字段 | 类型 | 必填 | 说明 | +| --- | --- | --- | --- | +| `query` | string | 是(`binding:"required"`) | 与单库 `q` 相同:Postgres 为 POSIX 正则(大小写不敏感);SQLite / Lite 为字面 `LIKE` 子串 | +| `knowledge_base_ids` | []string | 条件必填 | 多库;与 `knowledge_base_id` 至少提供一个,最多 32 个 | +| `knowledge_base_id` | string | 否 | 单库兼容字段,合并进 ids | +| `limit` | int | 否 | 全局 top-N,默认 10,最大 50 | + +任一目标库未开启 wiki → 400。库不存在或无读权限 → 404。API Key 白名单越权 → 403。非法正则(Postgres)→ 400。 + +响应:200 `{"success":true,"data":[WikiSearchHit]}`。命中只含导航字段 + `match_snippet`,不含正文;全文用 `GET /knowledgebase/:kb_id/wiki/pages/{slug}`。 + +| 响应字段 | 类型 | 说明 | +| --- | --- | --- | +| `id` | string | 页面 ID | +| `knowledge_base_id` | string | 来源库,跨库时用来拼读页 URL | +| `slug` | string | 页面 slug | +| `title` | string | 标题 | +| `page_type` | string | 页面类型 | +| `aliases` | []string | 别名 | +| `summary` | string | 摘要 | +| `match_snippet` | string | 正文中首次命中附近约 60+命中+60 字;无正文命中时省略。snippet 仍用 Go 正则编译 `query`;SQLite 下 `query` 含正则元字符时库内可命中、snippet 可能为空 | + +`content`、树/链接/元数据、时间戳不在搜索结果里。单库 `GET .../wiki/search` 仍返回完整 `WikiPage`。 + +```bash +curl -X POST $BASE/api/v1/wiki-search -H "X-API-Key: $API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"query":"部署|发布","knowledge_base_ids":["kb-1","kb-2"],"limit":10}' +``` + ### POST /api/v1/knowledgebase/:kb_id/wiki/rebuild-links 用途:重建页面互链。写权限。无请求体。