From fa67dfd44bc1ecef2d80a97841b62d0cd505a1b5 Mon Sep 17 00:00:00 2001 From: lyingbug <[email> Date: Tue, 4 Aug 2026 08:45:51 +0000 Subject: [PATCH] feat(knowledge): store folder path for folder uploads and expose folder tree API Folder uploads used to encode their relative directory inside file_name, which made the document list render the whole path as the title and left no way to browse or query a single folder. The directory now lives in a dedicated folder_path column (with a backfill for existing rows), the document list accepts folder_path / folder_recursive filters, and a new GET /knowledge-bases/:id/knowledge/folders endpoint returns the folder tree with per-folder document counts. --- client/knowledge.go | 1 + internal/application/repository/knowledge.go | 35 +++ .../repository/knowledge_finalize_test.go | 1 + .../repository/knowledge_folder_test.go | 115 ++++++++++ internal/application/service/knowledge.go | 14 ++ .../application/service/knowledge_create.go | 26 ++- .../service/knowledge_create_folder_test.go | 81 +++++++ .../application/service/knowledge_process.go | 1 + internal/handler/knowledge.go | 55 ++++- internal/handler/knowledge_folder_test.go | 147 ++++++++++++ internal/router/routes_knowledge.go | 1 + internal/types/interfaces/knowledge.go | 11 + internal/types/knowledge.go | 13 ++ internal/types/knowledge_folder.go | 213 ++++++++++++++++++ internal/types/knowledge_folder_test.go | 107 +++++++++ .../000002_knowledge_folder_path.down.sql | 8 + .../000002_knowledge_folder_path.up.sql | 23 ++ .../000079_knowledge_folder_path.down.sql | 9 + .../000079_knowledge_folder_path.up.sql | 21 ++ 19 files changed, 878 insertions(+), 4 deletions(-) create mode 100644 internal/application/repository/knowledge_folder_test.go create mode 100644 internal/application/service/knowledge_create_folder_test.go create mode 100644 internal/handler/knowledge_folder_test.go create mode 100644 internal/types/knowledge_folder.go create mode 100644 internal/types/knowledge_folder_test.go create mode 100644 migrations/sqlite/000002_knowledge_folder_path.down.sql create mode 100644 migrations/sqlite/000002_knowledge_folder_path.up.sql create mode 100644 migrations/versioned/000079_knowledge_folder_path.down.sql create mode 100644 migrations/versioned/000079_knowledge_folder_path.up.sql diff --git a/client/knowledge.go b/client/knowledge.go index 8bfbe299..7ab8d7de 100644 --- a/client/knowledge.go +++ b/client/knowledge.go @@ -36,6 +36,7 @@ type Knowledge struct { EnableStatus string `json:"enable_status"` EmbeddingModelID string `json:"embedding_model_id"` FileName string `json:"file_name"` + FolderPath string `json:"folder_path"` FileType string `json:"file_type"` FileSize int64 `json:"file_size"` FileHash string `json:"file_hash"` diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 9c17687b..581e3779 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -150,6 +150,20 @@ func applyKnowledgeListFilter(query *gorm.DB, filter types.KnowledgeListFilter) if !filter.UpdatedTo.IsZero() { query = query.Where("updated_at <= ?", filter.UpdatedTo) } + switch filter.FolderScope { + case types.FolderScopeExact: + query = query.Where("folder_path = ?", filter.FolderPath) + case types.FolderScopeSubtree: + // An empty path means "the whole knowledge base", so no predicate is + // needed; otherwise match the folder itself plus everything below it. + if filter.FolderPath != "" { + query = query.Where( + "(folder_path = ? OR folder_path LIKE ?)", + filter.FolderPath, + escapeLikeKeyword(filter.FolderPath)+"/%", + ) + } + } return query } @@ -186,6 +200,27 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID( return knowledges, total, nil } +// ListKnowledgeFolderCounts aggregates how many knowledge entries live directly +// in each folder of a knowledge base. Rows mid-deletion are excluded so the +// sidebar tree counts match the document list. +func (r *knowledgeRepository) ListKnowledgeFolderCounts( + ctx context.Context, + tenantID uint64, + kbID string, +) ([]*types.KnowledgeFolderCount, error) { + var counts []*types.KnowledgeFolderCount + if err := r.db.WithContext(ctx). + Model(&types.Knowledge{}). + Select("folder_path AS folder_path, COUNT(*) AS count"). + Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", + tenantID, kbID, types.ParseStatusDeleting). + Group("folder_path"). + Find(&counts).Error; err != nil { + return nil, err + } + return counts, nil +} + // UpdateKnowledge updates knowledge func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { omit := omitFieldsOnUpdate diff --git a/internal/application/repository/knowledge_finalize_test.go b/internal/application/repository/knowledge_finalize_test.go index cd1cc39b..b27efd1b 100644 --- a/internal/application/repository/knowledge_finalize_test.go +++ b/internal/application/repository/knowledge_finalize_test.go @@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS knowledges ( enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', embedding_model_id VARCHAR(64), file_name VARCHAR(255), + folder_path VARCHAR(1024) NOT NULL DEFAULT '', file_type VARCHAR(50), file_size BIGINT, file_path TEXT, diff --git a/internal/application/repository/knowledge_folder_test.go b/internal/application/repository/knowledge_folder_test.go new file mode 100644 index 00000000..b4111244 --- /dev/null +++ b/internal/application/repository/knowledge_folder_test.go @@ -0,0 +1,115 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// insertKnowledgeInFolder seeds a completed knowledge row stored in the given +// folder so the folder filter and folder aggregation can be asserted. +func insertKnowledgeInFolder(t *testing.T, db *gorm.DB, tenantID uint64, kbID, folderPath, fileName string) string { + t.Helper() + id := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, title, source, parse_status, file_name, folder_path) + VALUES (?, ?, ?, 'file', ?, 'manual', 'completed', ?, ?) + `, id, tenantID, kbID, fileName, fileName, folderPath).Error) + return id +} + +func TestListPagedKnowledgeFolderScope(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + const tenantID = uint64(1) + kbID := uuid.New().String() + + rootID := insertKnowledgeInFolder(t, db, tenantID, kbID, "", "readme.md") + docsID := insertKnowledgeInFolder(t, db, tenantID, kbID, "docs", "intro.md") + specID := insertKnowledgeInFolder(t, db, tenantID, kbID, "docs/spec", "design.md") + otherID := insertKnowledgeInFolder(t, db, tenantID, kbID, "assets", "logo.png") + + page := &types.Pagination{Page: 1, PageSize: 100} + list := func(filter types.KnowledgeListFilter) ([]string, int64) { + rows, total, err := repo.ListPagedKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID, page, filter) + require.NoError(t, err) + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + return ids, total + } + + // No folder scope: the flat listing is unchanged, which keeps every existing + // caller (and the "all documents" view) working. + ids, total := list(types.KnowledgeListFilter{}) + assert.ElementsMatch(t, []string{rootID, docsID, specID, otherID}, ids) + assert.Equal(t, int64(4), total) + + // Exact scope with an empty path means "knowledge base root only". + ids, total = list(types.KnowledgeListFilter{FolderScope: types.FolderScopeExact}) + assert.Equal(t, []string{rootID}, ids) + assert.Equal(t, int64(1), total) + + ids, total = list(types.KnowledgeListFilter{FolderPath: "docs", FolderScope: types.FolderScopeExact}) + assert.Equal(t, []string{docsID}, ids) + assert.Equal(t, int64(1), total) + + ids, total = list(types.KnowledgeListFilter{FolderPath: "docs", FolderScope: types.FolderScopeSubtree}) + assert.ElementsMatch(t, []string{docsID, specID}, ids) + assert.Equal(t, int64(2), total) + + // Subtree scope at the root spans the whole knowledge base. + ids, _ = list(types.KnowledgeListFilter{FolderScope: types.FolderScopeSubtree}) + assert.ElementsMatch(t, []string{rootID, docsID, specID, otherID}, ids) + + // Folder scope composes with the other filter dimensions. + ids, _ = list(types.KnowledgeListFilter{ + FolderPath: "docs", + FolderScope: types.FolderScopeSubtree, + Keyword: "design", + }) + assert.Equal(t, []string{specID}, ids) +} + +func TestListKnowledgeFolderCounts(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + const tenantID = uint64(1) + kbID := uuid.New().String() + otherKBID := uuid.New().String() + + insertKnowledgeInFolder(t, db, tenantID, kbID, "", "readme.md") + insertKnowledgeInFolder(t, db, tenantID, kbID, "docs/spec", "design.md") + insertKnowledgeInFolder(t, db, tenantID, kbID, "docs/spec", "api.md") + insertKnowledgeInFolder(t, db, tenantID, otherKBID, "docs/spec", "other-kb.md") + // Rows mid-deletion are excluded so the tree counts agree with the list. + deleting := insertKnowledgeInFolder(t, db, tenantID, kbID, "docs", "vanishing.md") + require.NoError(t, db.Exec(`UPDATE knowledges SET parse_status = ? WHERE id = ?`, + types.ParseStatusDeleting, deleting).Error) + + counts, err := repo.ListKnowledgeFolderCounts(ctx, tenantID, kbID) + require.NoError(t, err) + + byPath := map[string]int64{} + for _, row := range counts { + byPath[row.FolderPath] = row.Count + } + assert.Equal(t, map[string]int64{"": 1, "docs/spec": 2}, byPath) + + tree := types.BuildKnowledgeFolderTree(counts) + assert.Equal(t, int64(1), tree.RootDocumentCount) + assert.Equal(t, int64(3), tree.TotalDocumentCount) + require.Len(t, tree.Folders, 1) + assert.Equal(t, "docs", tree.Folders[0].Path) + assert.Equal(t, int64(2), tree.Folders[0].TotalCount) +} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index e633ba54..3fb3bcb9 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -564,6 +564,20 @@ func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Conte return types.NewPageResult(total, page, knowledges), nil } +// ListKnowledgeFolderTree returns the folder hierarchy of a knowledge base with +// per-folder document counts, derived from the folder_path stored on each +// knowledge entry. +func (s *knowledgeService) ListKnowledgeFolderTree(ctx context.Context, + kbID string, +) (*types.KnowledgeFolderTree, error) { + counts, err := s.repo.ListKnowledgeFolderCounts(ctx, + ctx.Value(types.TenantIDContextKey).(uint64), kbID) + if err != nil { + return nil, err + } + return types.BuildKnowledgeFolderTree(counts), nil +} + // GetKnowledgeFile retrieves the physical file associated with a knowledge entry func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) { // Get knowledge record diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index b7473c01..de52de61 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -28,11 +28,19 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from file") - // Use custom filename if provided, otherwise use original filename + // Use custom filename if provided, otherwise use original filename. Folder + // uploads pass a path-qualified name ("docs/spec/design.md"): the directory + // part becomes the knowledge's folder_path and only the base name is kept as + // the display / storage file name. fileName := file.Filename + folderPath := "" if customFileName != "" { - fileName = customFileName - logger.Infof(ctx, "Using custom filename: %s (original: %s)", customFileName, file.Filename) + folderPath, fileName = types.SplitKnowledgeRelativePath(customFileName) + if fileName == "" { + fileName = file.Filename + } + logger.Infof(ctx, "Using custom filename: %s (original: %s, folder: %s)", + fileName, file.Filename, folderPath) } logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName) @@ -125,6 +133,17 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, return nil, werrors.NewValidationError("文件名包含非法字符") } + // The folder path is rendered as sidebar tree labels, so it goes through the + // same input validation as the file name before it is stored. + if folderPath != "" { + safeFolderPath, folderValid := secutils.ValidateInput(folderPath) + if !folderValid { + logger.Errorf(ctx, "Invalid folder path: %s", folderPath) + return nil, werrors.NewValidationError("文件夹路径包含非法字符") + } + folderPath = types.NormalizeKnowledgeFolderPath(safeFolderPath) + } + eff, err := resolveFileImportProcessConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel) if err != nil { return nil, err @@ -140,6 +159,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, Channel: defaultChannel(channel), Title: safeFilename, FileName: safeFilename, + FolderPath: folderPath, FileType: getFileType(safeFilename), FileSize: file.Size, FileHash: hash, diff --git a/internal/application/service/knowledge_create_folder_test.go b/internal/application/service/knowledge_create_folder_test.go new file mode 100644 index 00000000..6479bd59 --- /dev/null +++ b/internal/application/service/knowledge_create_folder_test.go @@ -0,0 +1,81 @@ +package service + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +// createKnowledgeInFolder runs the file-upload path with the given +// path-qualified `fileName` (what the browser sends for a folder upload) and +// returns the persisted knowledge row. +func createKnowledgeInFolder(t *testing.T, uploadName, customFileName string) *types.Knowledge { + t.Helper() + + repo := &createKnowledgeFileRepoStub{} + svc := &knowledgeService{ + repo: repo, + kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}}, + fileSvc: &createKnowledgeFileServiceStub{}, + task: &createKnowledgeTaskEnqueuerStub{}, + } + + _, err := svc.CreateKnowledgeFromFile( + newCreateKnowledgeFileContext(), + "kb-1", + newMultipartFileHeader(t, uploadName, "hello"), + nil, + nil, + customFileName, + nil, + "", + nil, + ) + require.NoError(t, err) + require.NotNil(t, repo.createdKnowledge) + return repo.createdKnowledge +} + +// A folder upload sends the browser's webkitRelativePath as `fileName`. The +// directory part must land in folder_path while file_name keeps only the base +// name, so the document list shows "design.md" instead of the whole path. +func TestCreateKnowledgeFromFile_FolderUploadSplitsPathAndName(t *testing.T) { + t.Parallel() + + knowledge := createKnowledgeInFolder(t, "design.md", "docs/spec/design.md") + + require.Equal(t, "docs/spec", knowledge.FolderPath) + require.Equal(t, "design.md", knowledge.FileName) + require.Equal(t, "design.md", knowledge.Title) + require.Equal(t, "md", knowledge.FileType) +} + +func TestCreateKnowledgeFromFile_PlainUploadStaysAtRoot(t *testing.T) { + t.Parallel() + + knowledge := createKnowledgeInFolder(t, "doc.txt", "") + + require.Empty(t, knowledge.FolderPath, "a single-file upload belongs to the KB root") + require.Equal(t, "doc.txt", knowledge.FileName) +} + +// A custom file name without any separator keeps working as a pure rename. +func TestCreateKnowledgeFromFile_CustomNameWithoutFolder(t *testing.T) { + t.Parallel() + + knowledge := createKnowledgeInFolder(t, "upload.bin", "renamed.md") + + require.Empty(t, knowledge.FolderPath) + require.Equal(t, "renamed.md", knowledge.FileName) +} + +// Path traversal in the relative path must not escape the knowledge base root. +func TestCreateKnowledgeFromFile_FolderUploadRejectsTraversal(t *testing.T) { + t.Parallel() + + knowledge := createKnowledgeInFolder(t, "design.md", "../../etc/docs/design.md") + + require.Equal(t, "etc/docs", knowledge.FolderPath) + require.Equal(t, "design.md", knowledge.FileName) +} diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 354a3227..58161787 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -49,6 +49,7 @@ func (s *knowledgeService) cloneKnowledge( EnableStatus: "disabled", EmbeddingModelID: targetKB.EmbeddingModelID, FileName: src.FileName, + FolderPath: src.FolderPath, FileType: src.FileType, FileSize: src.FileSize, FileHash: src.FileHash, diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 7ad9cb26..d3644d3b 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -925,6 +925,8 @@ func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProces // @Param source query string false "来源/渠道筛选 (web/api/feishu/notion/yuque/wechat/...,或 manual/url 按 type 过滤)" // @Param start_time query string false "更新时间起点,RFC3339 格式" // @Param end_time query string false "更新时间终点,RFC3339 格式" +// @Param folder_path query string false "文件夹路径筛选,空字符串表示知识库根目录;不传该参数则不按文件夹过滤" +// @Param folder_recursive query bool false "为 true 时同时返回子文件夹内的文档" // @Success 200 {object} map[string]interface{} "知识列表" // @Failure 400 {object} errors.AppError "请求参数错误" // @Security Bearer @@ -976,10 +978,20 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) { } filter.UpdatedTo = t } + // The folder dimension is opt-in by parameter *presence*: an empty + // folder_path is meaningful (the knowledge base root), so it cannot be + // distinguished from "no folder filter" by value alone. + if raw, ok := c.GetQuery("folder_path"); ok { + filter.FolderPath = types.NormalizeKnowledgeFolderPath(raw) + filter.FolderScope = types.FolderScopeExact + if recursive, err := strconv.ParseBool(c.DefaultQuery("folder_recursive", "false")); err == nil && recursive { + filter.FolderScope = types.FolderScopeSubtree + } + } logger.Infof( ctx, - "Retrieving knowledge list under knowledge base, kb_id=%s tag_ids=%s keyword=%s file_type=%s parse_status=%s source=%s start_time=%s end_time=%s page=%d page_size=%d effectiveTenantID=%d", + "Retrieving knowledge list under knowledge base, kb_id=%s tag_ids=%s keyword=%s file_type=%s parse_status=%s source=%s start_time=%s end_time=%s folder_path=%s folder_scope=%s page=%d page_size=%d effectiveTenantID=%d", secutils.SanitizeForLog(kbID), secutils.SanitizeForLog(strings.Join(filter.TagIDs, ",")), secutils.SanitizeForLog(filter.Keyword), @@ -988,6 +1000,8 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) { secutils.SanitizeForLog(filter.Source), secutils.SanitizeForLog(c.Query("start_time")), secutils.SanitizeForLog(c.Query("end_time")), + secutils.SanitizeForLog(filter.FolderPath), + string(filter.FolderScope), pagination.Page, pagination.PageSize, effectiveTenantID, @@ -1016,6 +1030,45 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) { }) } +// ListKnowledgeFolders godoc +// @Summary 获取知识库文件夹目录树 +// @Description 返回知识库内由文件夹上传形成的目录树,包含每个文件夹的直接文档数与含子目录的总数 +// @Tags 知识管理 +// @Accept json +// @Produce json +// @Param id path string true "知识库ID" +// @Success 200 {object} map[string]interface{} "目录树" +// @Failure 400 {object} errors.AppError "请求参数错误" +// @Security Bearer +// @Security ApiKeyAuth +// @Router /knowledge-bases/{id}/knowledge/folders [get] +func (h *KnowledgeHandler) ListKnowledgeFolders(c *gin.Context) { + ctx := c.Request.Context() + + // Read access mirrors ListKnowledge so the sidebar tree is available to + // every viewer of a shared knowledge base. + _, kbID, effectiveTenantID, _, err := h.validateKnowledgeBaseAccess(c) + if err != nil { + c.Error(err) + return + } + ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + + tree, err := h.kgService.ListKnowledgeFolderTree(ctx, kbID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge folder tree retrieved, kb_id=%s folders=%d", + secutils.SanitizeForLog(kbID), len(tree.Folders)) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": tree, + }) +} + // DeleteKnowledge godoc // @Summary 删除知识 // @Description 根据ID异步删除知识条目。请求会被入队到与批量删除相同的异步管道(asynq); diff --git a/internal/handler/knowledge_folder_test.go b/internal/handler/knowledge_folder_test.go new file mode 100644 index 00000000..1510f89a --- /dev/null +++ b/internal/handler/knowledge_folder_test.go @@ -0,0 +1,147 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +type stubFolderKBService struct { + interfaces.KnowledgeBaseService +} + +func (s *stubFolderKBService) GetKnowledgeBaseByID(_ context.Context, id string) (*types.KnowledgeBase, error) { + if id != "kb-1" { + return nil, errors.New("kb not found") + } + return &types.KnowledgeBase{ID: "kb-1", TenantID: 1, Type: types.KnowledgeBaseTypeDocument}, nil +} + +type stubFolderKGService struct { + interfaces.KnowledgeService + + gotFilter types.KnowledgeListFilter + tree *types.KnowledgeFolderTree + treeErr error +} + +func (s *stubFolderKGService) ListPagedKnowledgeByKnowledgeBaseID( + _ context.Context, + _ string, + page *types.Pagination, + filter types.KnowledgeListFilter, +) (*types.PageResult, error) { + s.gotFilter = filter + return types.NewPageResult(0, page, []*types.Knowledge{}), nil +} + +func (s *stubFolderKGService) ListKnowledgeFolderTree( + _ context.Context, + _ string, +) (*types.KnowledgeFolderTree, error) { + return s.tree, s.treeErr +} + +func newFolderRouter(kg interfaces.KnowledgeService) *gin.Engine { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(middleware.ErrorHandler()) + r.Use(func(c *gin.Context) { + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Set(types.UserIDContextKey.String(), "u-test") + c.Next() + }) + h := &KnowledgeHandler{kbService: &stubFolderKBService{}, kgService: kg} + r.GET("/knowledge-bases/:id/knowledge", h.ListKnowledge) + r.GET("/knowledge-bases/:id/knowledge/folders", h.ListKnowledgeFolders) + return r +} + +// The folder dimension is keyed off parameter presence, because an empty +// folder_path is a meaningful value (the knowledge base root) and must be +// distinguishable from "browse every folder". +func TestListKnowledge_FolderScopeFromQuery(t *testing.T) { + tests := []struct { + name string + query string + wantPath string + wantScope types.KnowledgeFolderScope + }{ + {"absent parameter keeps the flat listing", "", "", types.FolderScopeAny}, + {"empty parameter selects the root", "?folder_path=", "", types.FolderScopeExact}, + {"folder selects that folder only", "?folder_path=docs/spec", "docs/spec", types.FolderScopeExact}, + {"recursive selects the subtree", "?folder_path=docs&folder_recursive=true", "docs", types.FolderScopeSubtree}, + {"recursive false stays exact", "?folder_path=docs&folder_recursive=false", "docs", types.FolderScopeExact}, + {"path is normalized", "?folder_path=/docs//spec/", "docs/spec", types.FolderScopeExact}, + {"traversal cannot escape the root", "?folder_path=../../spec", "spec", types.FolderScopeExact}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + kg := &stubFolderKGService{} + router := newFolderRouter(kg) + + req := httptest.NewRequest(http.MethodGet, "/knowledge-bases/kb-1/knowledge"+tt.query, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String()) + } + if kg.gotFilter.FolderPath != tt.wantPath { + t.Fatalf("folder path = %q, want %q", kg.gotFilter.FolderPath, tt.wantPath) + } + if kg.gotFilter.FolderScope != tt.wantScope { + t.Fatalf("folder scope = %q, want %q", kg.gotFilter.FolderScope, tt.wantScope) + } + }) + } +} + +func TestListKnowledgeFolders_ReturnsTree(t *testing.T) { + kg := &stubFolderKGService{ + tree: types.BuildKnowledgeFolderTree([]*types.KnowledgeFolderCount{ + {FolderPath: "", Count: 1}, + {FolderPath: "docs/spec", Count: 2}, + }), + } + router := newFolderRouter(kg) + + req := httptest.NewRequest(http.MethodGet, "/knowledge-bases/kb-1/knowledge/folders", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String()) + } + + var body struct { + Success bool `json:"success"` + Data types.KnowledgeFolderTree `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal response: %v; body=%s", err, w.Body.String()) + } + if !body.Success { + t.Fatalf("success = false; body=%s", w.Body.String()) + } + if body.Data.RootDocumentCount != 1 || body.Data.TotalDocumentCount != 3 { + t.Fatalf("counts = root %d total %d, want 1 / 3", + body.Data.RootDocumentCount, body.Data.TotalDocumentCount) + } + if len(body.Data.Folders) != 1 || body.Data.Folders[0].Path != "docs" { + t.Fatalf("folders = %+v, want a single \"docs\" root folder", body.Data.Folders) + } + if len(body.Data.Folders[0].Children) != 1 || body.Data.Folders[0].Children[0].Path != "docs/spec" { + t.Fatalf("children = %+v, want \"docs/spec\"", body.Data.Folders[0].Children) + } +} diff --git a/internal/router/routes_knowledge.go b/internal/router/routes_knowledge.go index 34e1dfe5..48d2337d 100644 --- a/internal/router/routes_knowledge.go +++ b/internal/router/routes_knowledge.go @@ -74,6 +74,7 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl kb.POST("/url", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateKnowledgeFromURL) kb.POST("/manual", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateManualKnowledge) kbRead.GET("", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledge) + kbRead.GET("/folders", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledgeFolders) // Clearing all contents under a KB is a destructive op; gate // behind Admin instead of Contributor. kb.With(apiKeyFullAccess()).DELETE("", g.Admin(), g.KBAccessWrite("id"), handler.ClearKnowledgeBaseContents) diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index daf86eca..388a8564 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -84,6 +84,10 @@ type KnowledgeService interface { page *types.Pagination, filter types.KnowledgeListFilter, ) (*types.PageResult, error) + // ListKnowledgeFolderTree returns the folder hierarchy derived from the + // folder_path of every knowledge entry in a knowledge base, with per-folder + // document counts. It powers the document sidebar tree. + ListKnowledgeFolderTree(ctx context.Context, kbID string) (*types.KnowledgeFolderTree, error) // DeleteKnowledge deletes knowledge by ID. DeleteKnowledge(ctx context.Context, id string) error // DeleteKnowledgeList deletes multiple knowledge entries by IDs. @@ -241,6 +245,13 @@ type KnowledgeRepository interface { kbID string, params *types.KnowledgeCheckParams, ) (bool, *types.Knowledge, error) + // ListKnowledgeFolderCounts aggregates the number of knowledge entries + // stored directly in each folder_path of a knowledge base. + ListKnowledgeFolderCounts( + ctx context.Context, + tenantID uint64, + kbID string, + ) ([]*types.KnowledgeFolderCount, error) // AminusB returns the IDs of knowledge in A that have no counterpart in B, // comparing file_hash as a multiset (so duplicate-count differences and // NULL/empty hashes are handled correctly, letting a clone converge). diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index f3955c47..7f5ac047 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -106,6 +106,13 @@ type KnowledgeListFilter struct { UpdatedFrom time.Time // UpdatedTo, when non-zero, keeps rows with updated_at <= UpdatedTo. UpdatedTo time.Time + // FolderPath is the folder navigated to in the sidebar tree. It is only + // applied when FolderScope is not FolderScopeAny, so the empty string can + // unambiguously mean "the knowledge base root". + FolderPath string + // FolderScope selects whether FolderPath matches exactly or includes + // descendant folders. FolderScopeAny (the default) ignores folders. + FolderScope KnowledgeFolderScope } // Knowledge represents a knowledge entity in the system. @@ -144,6 +151,12 @@ type Knowledge struct { EmbeddingModelID string `json:"embedding_model_id"` // File name of the knowledge FileName string `json:"file_name"` + // FolderPath is the canonical relative directory this entry belongs to + // inside the knowledge base, e.g. "docs/spec" for a folder upload of + // "docs/spec/design.md". Empty means the knowledge base root. It is a + // display/navigation concern only: it never affects where the file is + // physically stored (see FilePath). + FolderPath string `json:"folder_path" gorm:"type:varchar(1024);not null;default:''"` // File type of the knowledge FileType string `json:"file_type"` // File size of the knowledge diff --git a/internal/types/knowledge_folder.go b/internal/types/knowledge_folder.go new file mode 100644 index 00000000..3d314db7 --- /dev/null +++ b/internal/types/knowledge_folder.go @@ -0,0 +1,213 @@ +package types + +import ( + "sort" + "strings" +) + +const ( + // MaxKnowledgeFolderDepth caps how many nested levels a folder upload can + // create. Browsers happily hand us arbitrarily deep webkitRelativePath + // values; the cap keeps the sidebar tree renderable and bounds the stored + // path length. + MaxKnowledgeFolderDepth = 16 + // MaxKnowledgeFolderPathLength matches the folder_path column width. + MaxKnowledgeFolderPathLength = 1024 + // MaxKnowledgeFolderSegmentLength caps a single directory name so one + // pathological segment cannot consume the whole path budget. + MaxKnowledgeFolderSegmentLength = 128 +) + +// KnowledgeFolderScope selects how KnowledgeListFilter.FolderPath is applied. +type KnowledgeFolderScope string + +const ( + // FolderScopeAny ignores the folder dimension entirely (flat listing). + FolderScopeAny KnowledgeFolderScope = "" + // FolderScopeExact keeps only entries stored directly in FolderPath. + FolderScopeExact KnowledgeFolderScope = "exact" + // FolderScopeSubtree keeps entries in FolderPath and any descendant folder. + FolderScopeSubtree KnowledgeFolderScope = "subtree" +) + +// NormalizeKnowledgeFolderPath turns a client-supplied relative directory into +// the canonical form stored in knowledges.folder_path: forward-slash separated, +// no leading/trailing separator, no "." / ".." segments, depth and length +// capped. The zero value ("") means "knowledge base root". +func NormalizeKnowledgeFolderPath(raw string) string { + if raw == "" { + return "" + } + raw = strings.ReplaceAll(raw, "\\", "/") + segments := make([]string, 0, 8) + for _, segment := range strings.Split(raw, "/") { + segment = strings.TrimSpace(segment) + // Trailing dots and spaces are stripped so "a." and "a" cannot become + // two sibling folders that render identically. + segment = strings.TrimRight(segment, ". ") + if segment == "" || segment == "." || segment == ".." { + continue + } + if len(segment) > MaxKnowledgeFolderSegmentLength { + segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength]) + } + if segment == "" { + continue + } + segments = append(segments, segment) + if len(segments) >= MaxKnowledgeFolderDepth { + break + } + } + path := strings.Join(segments, "/") + for len(path) > MaxKnowledgeFolderPathLength && len(segments) > 0 { + segments = segments[:len(segments)-1] + path = strings.Join(segments, "/") + } + return path +} + +// SplitKnowledgeRelativePath splits an upload path such as +// "docs/spec/design.md" into its normalized folder path ("docs/spec") and its +// base file name ("design.md"). A plain file name yields an empty folder path, +// so callers can pass the raw `fileName` form field unconditionally. +func SplitKnowledgeRelativePath(raw string) (folderPath string, fileName string) { + normalized := strings.ReplaceAll(raw, "\\", "/") + idx := strings.LastIndex(normalized, "/") + if idx < 0 { + return "", strings.TrimSpace(normalized) + } + fileName = strings.TrimSpace(normalized[idx+1:]) + folderPath = NormalizeKnowledgeFolderPath(normalized[:idx]) + return folderPath, fileName +} + +// KnowledgeFolderName returns the display name of a folder path (its last +// segment). The root folder has no name. +func KnowledgeFolderName(path string) string { + if path == "" { + return "" + } + if idx := strings.LastIndex(path, "/"); idx >= 0 { + return path[idx+1:] + } + return path +} + +// KnowledgeFolderParent returns the parent folder path of the given path, or +// an empty string when the path is a root-level folder. +func KnowledgeFolderParent(path string) string { + if idx := strings.LastIndex(path, "/"); idx >= 0 { + return path[:idx] + } + return "" +} + +// KnowledgeFolderCount is one row of the folder aggregation query: the number +// of knowledge entries stored directly in a folder path. +type KnowledgeFolderCount struct { + FolderPath string `json:"folder_path" gorm:"column:folder_path"` + Count int64 `json:"count" gorm:"column:count"` +} + +// KnowledgeFolderNode is a node of the knowledge base folder tree exposed to +// clients. DocumentCount counts entries stored directly in this folder; +// TotalCount additionally counts every descendant folder's entries. +type KnowledgeFolderNode struct { + // Path is the canonical folder path, e.g. "docs/spec". + Path string `json:"path"` + // Name is the last segment of Path, used as the tree row label. + Name string `json:"name"` + // DocumentCount counts entries whose folder_path equals Path. + DocumentCount int64 `json:"document_count"` + // TotalCount counts entries in Path and in all descendant folders. + TotalCount int64 `json:"total_count"` + // Children are the direct sub-folders, sorted by name. + Children []*KnowledgeFolderNode `json:"children,omitempty"` +} + +// KnowledgeFolderTree is the response payload of the folder tree endpoint. +type KnowledgeFolderTree struct { + // RootDocumentCount counts entries that live at the knowledge base root + // (folder_path = ""), i.e. documents that were not uploaded as part of a + // folder. + RootDocumentCount int64 `json:"root_document_count"` + // TotalDocumentCount counts every entry in the knowledge base, matching + // the unfiltered document list total. + TotalDocumentCount int64 `json:"total_document_count"` + // Folders are the top-level folders, sorted by name. + Folders []*KnowledgeFolderNode `json:"folders"` +} + +// BuildKnowledgeFolderTree turns the flat per-folder counts returned by the +// repository into a tree. Intermediate folders that hold no documents of their +// own (only sub-folders) are materialized so the hierarchy stays connected. +func BuildKnowledgeFolderTree(counts []*KnowledgeFolderCount) *KnowledgeFolderTree { + tree := &KnowledgeFolderTree{Folders: []*KnowledgeFolderNode{}} + nodes := map[string]*KnowledgeFolderNode{} + children := map[string][]*KnowledgeFolderNode{} + + // ensure materializes a folder node and every missing ancestor. + var ensure func(path string) *KnowledgeFolderNode + ensure = func(path string) *KnowledgeFolderNode { + if node, ok := nodes[path]; ok { + return node + } + node := &KnowledgeFolderNode{Path: path, Name: KnowledgeFolderName(path)} + nodes[path] = node + parent := KnowledgeFolderParent(path) + if parent == "" { + tree.Folders = append(tree.Folders, node) + } else { + ensure(parent) + children[parent] = append(children[parent], node) + } + return node + } + + for _, row := range counts { + if row == nil { + continue + } + tree.TotalDocumentCount += row.Count + path := NormalizeKnowledgeFolderPath(row.FolderPath) + if path == "" { + tree.RootDocumentCount += row.Count + continue + } + ensure(path).DocumentCount += row.Count + } + + // Roll descendant counts up. Deepest paths first so a parent always sees + // final child totals. + paths := make([]string, 0, len(nodes)) + for path := range nodes { + paths = append(paths, path) + } + sort.Slice(paths, func(i, j int) bool { + di, dj := strings.Count(paths[i], "/"), strings.Count(paths[j], "/") + if di != dj { + return di > dj + } + return paths[i] < paths[j] + }) + for _, path := range paths { + node := nodes[path] + node.TotalCount += node.DocumentCount + if parent := KnowledgeFolderParent(path); parent != "" { + nodes[parent].TotalCount += node.TotalCount + } + } + + sortNodes := func(list []*KnowledgeFolderNode) { + sort.Slice(list, func(i, j int) bool { + return strings.ToLower(list[i].Name) < strings.ToLower(list[j].Name) + }) + } + for path, list := range children { + sortNodes(list) + nodes[path].Children = list + } + sortNodes(tree.Folders) + return tree +} diff --git a/internal/types/knowledge_folder_test.go b/internal/types/knowledge_folder_test.go new file mode 100644 index 00000000..f45ee62a --- /dev/null +++ b/internal/types/knowledge_folder_test.go @@ -0,0 +1,107 @@ +package types + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeKnowledgeFolderPath(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty stays root", "", ""}, + {"plain path", "docs/spec", "docs/spec"}, + {"windows separators", `docs\spec`, "docs/spec"}, + {"leading and trailing separators", "/docs/spec/", "docs/spec"}, + {"collapses empty segments", "docs//spec", "docs/spec"}, + {"drops traversal segments", "docs/../../etc/passwd", "docs/etc/passwd"}, + {"drops dot segments", "./docs/./spec", "docs/spec"}, + {"trims whitespace and trailing dots", " docs . / spec ", "docs/spec"}, + {"root only separators", "///", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, NormalizeKnowledgeFolderPath(tc.in)) + }) + } +} + +func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) { + deep := strings.Repeat("a/", MaxKnowledgeFolderDepth+5) + "a" + normalized := NormalizeKnowledgeFolderPath(deep) + assert.Equal(t, MaxKnowledgeFolderDepth, strings.Count(normalized, "/")+1, + "depth must be capped so the sidebar tree stays renderable") + + longSegment := strings.Repeat("x", MaxKnowledgeFolderSegmentLength+50) + assert.Len(t, NormalizeKnowledgeFolderPath(longSegment), MaxKnowledgeFolderSegmentLength) + + wide := strings.TrimSuffix(strings.Repeat(strings.Repeat("y", 100)+"/", 12), "/") + assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength) +} + +func TestSplitKnowledgeRelativePath(t *testing.T) { + cases := []struct { + in string + wantFolder string + wantName string + }{ + {"design.md", "", "design.md"}, + {"docs/spec/design.md", "docs/spec", "design.md"}, + {`docs\spec\design.md`, "docs/spec", "design.md"}, + {"/docs/design.md", "docs", "design.md"}, + {"docs/../design.md", "docs", "design.md"}, + {"docs/", "docs", ""}, + } + for _, tc := range cases { + folder, name := SplitKnowledgeRelativePath(tc.in) + assert.Equal(t, tc.wantFolder, folder, "folder for %q", tc.in) + assert.Equal(t, tc.wantName, name, "name for %q", tc.in) + } +} + +func TestBuildKnowledgeFolderTree(t *testing.T) { + tree := BuildKnowledgeFolderTree([]*KnowledgeFolderCount{ + {FolderPath: "", Count: 2}, + {FolderPath: "docs/spec", Count: 3}, + {FolderPath: "docs/spec/v2", Count: 1}, + {FolderPath: "Assets", Count: 4}, + }) + + assert.Equal(t, int64(2), tree.RootDocumentCount) + assert.Equal(t, int64(10), tree.TotalDocumentCount) + + // Top level is sorted case-insensitively: "Assets" before "docs". + require.Len(t, tree.Folders, 2) + assert.Equal(t, "Assets", tree.Folders[0].Path) + assert.Equal(t, int64(4), tree.Folders[0].TotalCount) + + docs := tree.Folders[1] + assert.Equal(t, "docs", docs.Path) + // "docs" itself holds no document but is materialized to keep the tree + // connected, and its total rolls up both descendants. + assert.Equal(t, int64(0), docs.DocumentCount) + assert.Equal(t, int64(4), docs.TotalCount) + + require.Len(t, docs.Children, 1) + spec := docs.Children[0] + assert.Equal(t, "docs/spec", spec.Path) + assert.Equal(t, "spec", spec.Name) + assert.Equal(t, int64(3), spec.DocumentCount) + assert.Equal(t, int64(4), spec.TotalCount) + + require.Len(t, spec.Children, 1) + assert.Equal(t, "docs/spec/v2", spec.Children[0].Path) + assert.Equal(t, int64(1), spec.Children[0].TotalCount) +} + +func TestBuildKnowledgeFolderTreeEmpty(t *testing.T) { + tree := BuildKnowledgeFolderTree(nil) + assert.Equal(t, int64(0), tree.TotalDocumentCount) + assert.NotNil(t, tree.Folders, "folders must serialize as [] rather than null") + assert.Empty(t, tree.Folders) +} diff --git a/migrations/sqlite/000002_knowledge_folder_path.down.sql b/migrations/sqlite/000002_knowledge_folder_path.down.sql new file mode 100644 index 00000000..e5a1cb78 --- /dev/null +++ b/migrations/sqlite/000002_knowledge_folder_path.down.sql @@ -0,0 +1,8 @@ +-- Fold the relative directory back into file_name before dropping the column. +UPDATE knowledges +SET file_name = folder_path || '/' || file_name +WHERE folder_path <> '' AND file_name <> ''; + +DROP INDEX IF EXISTS idx_knowledges_folder_path; + +ALTER TABLE knowledges DROP COLUMN folder_path; diff --git a/migrations/sqlite/000002_knowledge_folder_path.up.sql b/migrations/sqlite/000002_knowledge_folder_path.up.sql new file mode 100644 index 00000000..7f2d7c17 --- /dev/null +++ b/migrations/sqlite/000002_knowledge_folder_path.up.sql @@ -0,0 +1,23 @@ +-- Folder tree support for knowledge bases (SQLite / lite mode). +-- +-- Mirrors migrations/versioned/000079_knowledge_folder_path.up.sql. +ALTER TABLE knowledges ADD COLUMN folder_path VARCHAR(1024) NOT NULL DEFAULT ''; + +-- Backfill legacy folder uploads, which encoded their relative directory in +-- file_name. SQLite has no regex, so the directory prefix is isolated with the +-- rtrim trick: trimming every non-slash character off the right leaves exactly +-- the prefix up to and including the last '/'. +UPDATE knowledges +SET folder_path = substr( + rtrim(file_name, replace(file_name, '/', '')), + 1, + length(rtrim(file_name, replace(file_name, '/', ''))) - 1 + ), + file_name = substr( + file_name, + length(rtrim(file_name, replace(file_name, '/', ''))) + 1 + ) +WHERE file_name LIKE '%/%'; + +CREATE INDEX IF NOT EXISTS idx_knowledges_folder_path + ON knowledges (tenant_id, knowledge_base_id, folder_path); diff --git a/migrations/versioned/000079_knowledge_folder_path.down.sql b/migrations/versioned/000079_knowledge_folder_path.down.sql new file mode 100644 index 00000000..b5d8554c --- /dev/null +++ b/migrations/versioned/000079_knowledge_folder_path.down.sql @@ -0,0 +1,9 @@ +-- Restore the legacy representation: the relative directory is folded back +-- into file_name so a rollback keeps folder uploads distinguishable. +UPDATE knowledges +SET file_name = folder_path || '/' || file_name +WHERE folder_path <> '' AND file_name <> ''; + +DROP INDEX IF EXISTS idx_knowledges_folder_path; + +ALTER TABLE knowledges DROP COLUMN IF EXISTS folder_path; diff --git a/migrations/versioned/000079_knowledge_folder_path.up.sql b/migrations/versioned/000079_knowledge_folder_path.up.sql new file mode 100644 index 00000000..9b38f60a --- /dev/null +++ b/migrations/versioned/000079_knowledge_folder_path.up.sql @@ -0,0 +1,21 @@ +-- Folder tree support for knowledge bases. +-- +-- Folder uploads used to encode their relative directory inside file_name +-- (e.g. "docs/spec/design.md"), which made the document list show the whole +-- path as the title and left no way to query a single folder. The path now +-- lives in its own column and file_name keeps only the base name. +ALTER TABLE knowledges + ADD COLUMN IF NOT EXISTS folder_path VARCHAR(1024) NOT NULL DEFAULT ''; + +-- Backfill legacy folder uploads so existing knowledge bases render the same +-- tree as newly uploaded folders. +UPDATE knowledges +SET folder_path = LEFT( + REGEXP_REPLACE(file_name, '/[^/]*$', ''), + 1024 + ), + file_name = REGEXP_REPLACE(file_name, '^.*/', '') +WHERE file_name LIKE '%/%'; + +CREATE INDEX IF NOT EXISTS idx_knowledges_folder_path + ON knowledges (tenant_id, knowledge_base_id, folder_path);