feat(knowledge): store folder path for folder uploads and expose folder tree API

Folder uploads used to encode their relative directory inside file_name,
which made the document list render the whole path as the title and left
no way to browse or query a single folder. The directory now lives in a
dedicated folder_path column (with a backfill for existing rows), the
document list accepts folder_path / folder_recursive filters, and a new
GET /knowledge-bases/:id/knowledge/folders endpoint returns the folder
tree with per-folder document counts.
main
lyingbug 2026-08-04 08:45:51 +00:00 committed by lyingbug
parent 2b31d3681b
commit fa67dfd44b
19 changed files with 878 additions and 4 deletions

View File

@ -36,6 +36,7 @@ type Knowledge struct {
EnableStatus string `json:"enable_status"`
EmbeddingModelID string `json:"embedding_model_id"`
FileName string `json:"file_name"`
FolderPath string `json:"folder_path"`
FileType string `json:"file_type"`
FileSize int64 `json:"file_size"`
FileHash string `json:"file_hash"`

View File

@ -150,6 +150,20 @@ func applyKnowledgeListFilter(query *gorm.DB, filter types.KnowledgeListFilter)
if !filter.UpdatedTo.IsZero() {
query = query.Where("updated_at <= ?", filter.UpdatedTo)
}
switch filter.FolderScope {
case types.FolderScopeExact:
query = query.Where("folder_path = ?", filter.FolderPath)
case types.FolderScopeSubtree:
// An empty path means "the whole knowledge base", so no predicate is
// needed; otherwise match the folder itself plus everything below it.
if filter.FolderPath != "" {
query = query.Where(
"(folder_path = ? OR folder_path LIKE ?)",
filter.FolderPath,
escapeLikeKeyword(filter.FolderPath)+"/%",
)
}
}
return query
}
@ -186,6 +200,27 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID(
return knowledges, total, nil
}
// ListKnowledgeFolderCounts aggregates how many knowledge entries live directly
// in each folder of a knowledge base. Rows mid-deletion are excluded so the
// sidebar tree counts match the document list.
func (r *knowledgeRepository) ListKnowledgeFolderCounts(
ctx context.Context,
tenantID uint64,
kbID string,
) ([]*types.KnowledgeFolderCount, error) {
var counts []*types.KnowledgeFolderCount
if err := r.db.WithContext(ctx).
Model(&types.Knowledge{}).
Select("folder_path AS folder_path, COUNT(*) AS count").
Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?",
tenantID, kbID, types.ParseStatusDeleting).
Group("folder_path").
Find(&counts).Error; err != nil {
return nil, err
}
return counts, nil
}
// UpdateKnowledge updates knowledge
func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error {
omit := omitFieldsOnUpdate

View File

@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS knowledges (
enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
embedding_model_id VARCHAR(64),
file_name VARCHAR(255),
folder_path VARCHAR(1024) NOT NULL DEFAULT '',
file_type VARCHAR(50),
file_size BIGINT,
file_path TEXT,

View File

@ -0,0 +1,115 @@
package repository
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
// insertKnowledgeInFolder seeds a completed knowledge row stored in the given
// folder so the folder filter and folder aggregation can be asserted.
func insertKnowledgeInFolder(t *testing.T, db *gorm.DB, tenantID uint64, kbID, folderPath, fileName string) string {
t.Helper()
id := uuid.New().String()
require.NoError(t, db.Exec(`
INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, title, source, parse_status, file_name, folder_path)
VALUES (?, ?, ?, 'file', ?, 'manual', 'completed', ?, ?)
`, id, tenantID, kbID, fileName, fileName, folderPath).Error)
return id
}
func TestListPagedKnowledgeFolderScope(t *testing.T) {
db := setupKnowledgeTestDB(t)
repo := NewKnowledgeRepository(db).(*knowledgeRepository)
ctx := context.Background()
const tenantID = uint64(1)
kbID := uuid.New().String()
rootID := insertKnowledgeInFolder(t, db, tenantID, kbID, "", "readme.md")
docsID := insertKnowledgeInFolder(t, db, tenantID, kbID, "docs", "intro.md")
specID := insertKnowledgeInFolder(t, db, tenantID, kbID, "docs/spec", "design.md")
otherID := insertKnowledgeInFolder(t, db, tenantID, kbID, "assets", "logo.png")
page := &types.Pagination{Page: 1, PageSize: 100}
list := func(filter types.KnowledgeListFilter) ([]string, int64) {
rows, total, err := repo.ListPagedKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID, page, filter)
require.NoError(t, err)
ids := make([]string, 0, len(rows))
for _, row := range rows {
ids = append(ids, row.ID)
}
return ids, total
}
// No folder scope: the flat listing is unchanged, which keeps every existing
// caller (and the "all documents" view) working.
ids, total := list(types.KnowledgeListFilter{})
assert.ElementsMatch(t, []string{rootID, docsID, specID, otherID}, ids)
assert.Equal(t, int64(4), total)
// Exact scope with an empty path means "knowledge base root only".
ids, total = list(types.KnowledgeListFilter{FolderScope: types.FolderScopeExact})
assert.Equal(t, []string{rootID}, ids)
assert.Equal(t, int64(1), total)
ids, total = list(types.KnowledgeListFilter{FolderPath: "docs", FolderScope: types.FolderScopeExact})
assert.Equal(t, []string{docsID}, ids)
assert.Equal(t, int64(1), total)
ids, total = list(types.KnowledgeListFilter{FolderPath: "docs", FolderScope: types.FolderScopeSubtree})
assert.ElementsMatch(t, []string{docsID, specID}, ids)
assert.Equal(t, int64(2), total)
// Subtree scope at the root spans the whole knowledge base.
ids, _ = list(types.KnowledgeListFilter{FolderScope: types.FolderScopeSubtree})
assert.ElementsMatch(t, []string{rootID, docsID, specID, otherID}, ids)
// Folder scope composes with the other filter dimensions.
ids, _ = list(types.KnowledgeListFilter{
FolderPath: "docs",
FolderScope: types.FolderScopeSubtree,
Keyword: "design",
})
assert.Equal(t, []string{specID}, ids)
}
func TestListKnowledgeFolderCounts(t *testing.T) {
db := setupKnowledgeTestDB(t)
repo := NewKnowledgeRepository(db).(*knowledgeRepository)
ctx := context.Background()
const tenantID = uint64(1)
kbID := uuid.New().String()
otherKBID := uuid.New().String()
insertKnowledgeInFolder(t, db, tenantID, kbID, "", "readme.md")
insertKnowledgeInFolder(t, db, tenantID, kbID, "docs/spec", "design.md")
insertKnowledgeInFolder(t, db, tenantID, kbID, "docs/spec", "api.md")
insertKnowledgeInFolder(t, db, tenantID, otherKBID, "docs/spec", "other-kb.md")
// Rows mid-deletion are excluded so the tree counts agree with the list.
deleting := insertKnowledgeInFolder(t, db, tenantID, kbID, "docs", "vanishing.md")
require.NoError(t, db.Exec(`UPDATE knowledges SET parse_status = ? WHERE id = ?`,
types.ParseStatusDeleting, deleting).Error)
counts, err := repo.ListKnowledgeFolderCounts(ctx, tenantID, kbID)
require.NoError(t, err)
byPath := map[string]int64{}
for _, row := range counts {
byPath[row.FolderPath] = row.Count
}
assert.Equal(t, map[string]int64{"": 1, "docs/spec": 2}, byPath)
tree := types.BuildKnowledgeFolderTree(counts)
assert.Equal(t, int64(1), tree.RootDocumentCount)
assert.Equal(t, int64(3), tree.TotalDocumentCount)
require.Len(t, tree.Folders, 1)
assert.Equal(t, "docs", tree.Folders[0].Path)
assert.Equal(t, int64(2), tree.Folders[0].TotalCount)
}

View File

@ -564,6 +564,20 @@ func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Conte
return types.NewPageResult(total, page, knowledges), nil
}
// ListKnowledgeFolderTree returns the folder hierarchy of a knowledge base with
// per-folder document counts, derived from the folder_path stored on each
// knowledge entry.
func (s *knowledgeService) ListKnowledgeFolderTree(ctx context.Context,
kbID string,
) (*types.KnowledgeFolderTree, error) {
counts, err := s.repo.ListKnowledgeFolderCounts(ctx,
ctx.Value(types.TenantIDContextKey).(uint64), kbID)
if err != nil {
return nil, err
}
return types.BuildKnowledgeFolderTree(counts), nil
}
// GetKnowledgeFile retrieves the physical file associated with a knowledge entry
func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) {
// Get knowledge record

View File

@ -28,11 +28,19 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file")
// Use custom filename if provided, otherwise use original filename
// Use custom filename if provided, otherwise use original filename. Folder
// uploads pass a path-qualified name ("docs/spec/design.md"): the directory
// part becomes the knowledge's folder_path and only the base name is kept as
// the display / storage file name.
fileName := file.Filename
folderPath := ""
if customFileName != "" {
fileName = customFileName
logger.Infof(ctx, "Using custom filename: %s (original: %s)", customFileName, file.Filename)
folderPath, fileName = types.SplitKnowledgeRelativePath(customFileName)
if fileName == "" {
fileName = file.Filename
}
logger.Infof(ctx, "Using custom filename: %s (original: %s, folder: %s)",
fileName, file.Filename, folderPath)
}
logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, fileName)
@ -125,6 +133,17 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, werrors.NewValidationError("文件名包含非法字符")
}
// The folder path is rendered as sidebar tree labels, so it goes through the
// same input validation as the file name before it is stored.
if folderPath != "" {
safeFolderPath, folderValid := secutils.ValidateInput(folderPath)
if !folderValid {
logger.Errorf(ctx, "Invalid folder path: %s", folderPath)
return nil, werrors.NewValidationError("文件夹路径包含非法字符")
}
folderPath = types.NormalizeKnowledgeFolderPath(safeFolderPath)
}
eff, err := resolveFileImportProcessConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel)
if err != nil {
return nil, err
@ -140,6 +159,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
Channel: defaultChannel(channel),
Title: safeFilename,
FileName: safeFilename,
FolderPath: folderPath,
FileType: getFileType(safeFilename),
FileSize: file.Size,
FileHash: hash,

View File

@ -0,0 +1,81 @@
package service
import (
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
// createKnowledgeInFolder runs the file-upload path with the given
// path-qualified `fileName` (what the browser sends for a folder upload) and
// returns the persisted knowledge row.
func createKnowledgeInFolder(t *testing.T, uploadName, customFileName string) *types.Knowledge {
t.Helper()
repo := &createKnowledgeFileRepoStub{}
svc := &knowledgeService{
repo: repo,
kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}},
fileSvc: &createKnowledgeFileServiceStub{},
task: &createKnowledgeTaskEnqueuerStub{},
}
_, err := svc.CreateKnowledgeFromFile(
newCreateKnowledgeFileContext(),
"kb-1",
newMultipartFileHeader(t, uploadName, "hello"),
nil,
nil,
customFileName,
nil,
"",
nil,
)
require.NoError(t, err)
require.NotNil(t, repo.createdKnowledge)
return repo.createdKnowledge
}
// A folder upload sends the browser's webkitRelativePath as `fileName`. The
// directory part must land in folder_path while file_name keeps only the base
// name, so the document list shows "design.md" instead of the whole path.
func TestCreateKnowledgeFromFile_FolderUploadSplitsPathAndName(t *testing.T) {
t.Parallel()
knowledge := createKnowledgeInFolder(t, "design.md", "docs/spec/design.md")
require.Equal(t, "docs/spec", knowledge.FolderPath)
require.Equal(t, "design.md", knowledge.FileName)
require.Equal(t, "design.md", knowledge.Title)
require.Equal(t, "md", knowledge.FileType)
}
func TestCreateKnowledgeFromFile_PlainUploadStaysAtRoot(t *testing.T) {
t.Parallel()
knowledge := createKnowledgeInFolder(t, "doc.txt", "")
require.Empty(t, knowledge.FolderPath, "a single-file upload belongs to the KB root")
require.Equal(t, "doc.txt", knowledge.FileName)
}
// A custom file name without any separator keeps working as a pure rename.
func TestCreateKnowledgeFromFile_CustomNameWithoutFolder(t *testing.T) {
t.Parallel()
knowledge := createKnowledgeInFolder(t, "upload.bin", "renamed.md")
require.Empty(t, knowledge.FolderPath)
require.Equal(t, "renamed.md", knowledge.FileName)
}
// Path traversal in the relative path must not escape the knowledge base root.
func TestCreateKnowledgeFromFile_FolderUploadRejectsTraversal(t *testing.T) {
t.Parallel()
knowledge := createKnowledgeInFolder(t, "design.md", "../../etc/docs/design.md")
require.Equal(t, "etc/docs", knowledge.FolderPath)
require.Equal(t, "design.md", knowledge.FileName)
}

View File

@ -49,6 +49,7 @@ func (s *knowledgeService) cloneKnowledge(
EnableStatus: "disabled",
EmbeddingModelID: targetKB.EmbeddingModelID,
FileName: src.FileName,
FolderPath: src.FolderPath,
FileType: src.FileType,
FileSize: src.FileSize,
FileHash: src.FileHash,

View File

@ -925,6 +925,8 @@ func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProces
// @Param source query string false "来源/渠道筛选 (web/api/feishu/notion/yuque/wechat/...,或 manual/url 按 type 过滤)"
// @Param start_time query string false "更新时间起点,RFC3339 格式"
// @Param end_time query string false "更新时间终点,RFC3339 格式"
// @Param folder_path query string false "文件夹路径筛选,空字符串表示知识库根目录;不传该参数则不按文件夹过滤"
// @Param folder_recursive query bool false "为 true 时同时返回子文件夹内的文档"
// @Success 200 {object} map[string]interface{} "知识列表"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
@ -976,10 +978,20 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) {
}
filter.UpdatedTo = t
}
// The folder dimension is opt-in by parameter *presence*: an empty
// folder_path is meaningful (the knowledge base root), so it cannot be
// distinguished from "no folder filter" by value alone.
if raw, ok := c.GetQuery("folder_path"); ok {
filter.FolderPath = types.NormalizeKnowledgeFolderPath(raw)
filter.FolderScope = types.FolderScopeExact
if recursive, err := strconv.ParseBool(c.DefaultQuery("folder_recursive", "false")); err == nil && recursive {
filter.FolderScope = types.FolderScopeSubtree
}
}
logger.Infof(
ctx,
"Retrieving knowledge list under knowledge base, kb_id=%s tag_ids=%s keyword=%s file_type=%s parse_status=%s source=%s start_time=%s end_time=%s page=%d page_size=%d effectiveTenantID=%d",
"Retrieving knowledge list under knowledge base, kb_id=%s tag_ids=%s keyword=%s file_type=%s parse_status=%s source=%s start_time=%s end_time=%s folder_path=%s folder_scope=%s page=%d page_size=%d effectiveTenantID=%d",
secutils.SanitizeForLog(kbID),
secutils.SanitizeForLog(strings.Join(filter.TagIDs, ",")),
secutils.SanitizeForLog(filter.Keyword),
@ -988,6 +1000,8 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) {
secutils.SanitizeForLog(filter.Source),
secutils.SanitizeForLog(c.Query("start_time")),
secutils.SanitizeForLog(c.Query("end_time")),
secutils.SanitizeForLog(filter.FolderPath),
string(filter.FolderScope),
pagination.Page,
pagination.PageSize,
effectiveTenantID,
@ -1016,6 +1030,45 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) {
})
}
// ListKnowledgeFolders godoc
// @Summary 获取知识库文件夹目录树
// @Description 返回知识库内由文件夹上传形成的目录树,包含每个文件夹的直接文档数与含子目录的总数
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Success 200 {object} map[string]interface{} "目录树"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/folders [get]
func (h *KnowledgeHandler) ListKnowledgeFolders(c *gin.Context) {
ctx := c.Request.Context()
// Read access mirrors ListKnowledge so the sidebar tree is available to
// every viewer of a shared knowledge base.
_, kbID, effectiveTenantID, _, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
tree, err := h.kgService.ListKnowledgeFolderTree(ctx, kbID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge folder tree retrieved, kb_id=%s folders=%d",
secutils.SanitizeForLog(kbID), len(tree.Folders))
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": tree,
})
}
// DeleteKnowledge godoc
// @Summary 删除知识
// @Description 根据ID异步删除知识条目。请求会被入队到与批量删除相同的异步管道(asynq);

View File

@ -0,0 +1,147 @@
package handler
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/Tencent/WeKnora/internal/middleware"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
type stubFolderKBService struct {
interfaces.KnowledgeBaseService
}
func (s *stubFolderKBService) GetKnowledgeBaseByID(_ context.Context, id string) (*types.KnowledgeBase, error) {
if id != "kb-1" {
return nil, errors.New("kb not found")
}
return &types.KnowledgeBase{ID: "kb-1", TenantID: 1, Type: types.KnowledgeBaseTypeDocument}, nil
}
type stubFolderKGService struct {
interfaces.KnowledgeService
gotFilter types.KnowledgeListFilter
tree *types.KnowledgeFolderTree
treeErr error
}
func (s *stubFolderKGService) ListPagedKnowledgeByKnowledgeBaseID(
_ context.Context,
_ string,
page *types.Pagination,
filter types.KnowledgeListFilter,
) (*types.PageResult, error) {
s.gotFilter = filter
return types.NewPageResult(0, page, []*types.Knowledge{}), nil
}
func (s *stubFolderKGService) ListKnowledgeFolderTree(
_ context.Context,
_ string,
) (*types.KnowledgeFolderTree, error) {
return s.tree, s.treeErr
}
func newFolderRouter(kg interfaces.KnowledgeService) *gin.Engine {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(middleware.ErrorHandler())
r.Use(func(c *gin.Context) {
c.Set(types.TenantIDContextKey.String(), uint64(1))
c.Set(types.UserIDContextKey.String(), "u-test")
c.Next()
})
h := &KnowledgeHandler{kbService: &stubFolderKBService{}, kgService: kg}
r.GET("/knowledge-bases/:id/knowledge", h.ListKnowledge)
r.GET("/knowledge-bases/:id/knowledge/folders", h.ListKnowledgeFolders)
return r
}
// The folder dimension is keyed off parameter presence, because an empty
// folder_path is a meaningful value (the knowledge base root) and must be
// distinguishable from "browse every folder".
func TestListKnowledge_FolderScopeFromQuery(t *testing.T) {
tests := []struct {
name string
query string
wantPath string
wantScope types.KnowledgeFolderScope
}{
{"absent parameter keeps the flat listing", "", "", types.FolderScopeAny},
{"empty parameter selects the root", "?folder_path=", "", types.FolderScopeExact},
{"folder selects that folder only", "?folder_path=docs/spec", "docs/spec", types.FolderScopeExact},
{"recursive selects the subtree", "?folder_path=docs&folder_recursive=true", "docs", types.FolderScopeSubtree},
{"recursive false stays exact", "?folder_path=docs&folder_recursive=false", "docs", types.FolderScopeExact},
{"path is normalized", "?folder_path=/docs//spec/", "docs/spec", types.FolderScopeExact},
{"traversal cannot escape the root", "?folder_path=../../spec", "spec", types.FolderScopeExact},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
kg := &stubFolderKGService{}
router := newFolderRouter(kg)
req := httptest.NewRequest(http.MethodGet, "/knowledge-bases/kb-1/knowledge"+tt.query, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String())
}
if kg.gotFilter.FolderPath != tt.wantPath {
t.Fatalf("folder path = %q, want %q", kg.gotFilter.FolderPath, tt.wantPath)
}
if kg.gotFilter.FolderScope != tt.wantScope {
t.Fatalf("folder scope = %q, want %q", kg.gotFilter.FolderScope, tt.wantScope)
}
})
}
}
func TestListKnowledgeFolders_ReturnsTree(t *testing.T) {
kg := &stubFolderKGService{
tree: types.BuildKnowledgeFolderTree([]*types.KnowledgeFolderCount{
{FolderPath: "", Count: 1},
{FolderPath: "docs/spec", Count: 2},
}),
}
router := newFolderRouter(kg)
req := httptest.NewRequest(http.MethodGet, "/knowledge-bases/kb-1/knowledge/folders", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String())
}
var body struct {
Success bool `json:"success"`
Data types.KnowledgeFolderTree `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("unmarshal response: %v; body=%s", err, w.Body.String())
}
if !body.Success {
t.Fatalf("success = false; body=%s", w.Body.String())
}
if body.Data.RootDocumentCount != 1 || body.Data.TotalDocumentCount != 3 {
t.Fatalf("counts = root %d total %d, want 1 / 3",
body.Data.RootDocumentCount, body.Data.TotalDocumentCount)
}
if len(body.Data.Folders) != 1 || body.Data.Folders[0].Path != "docs" {
t.Fatalf("folders = %+v, want a single \"docs\" root folder", body.Data.Folders)
}
if len(body.Data.Folders[0].Children) != 1 || body.Data.Folders[0].Children[0].Path != "docs/spec" {
t.Fatalf("children = %+v, want \"docs/spec\"", body.Data.Folders[0].Children)
}
}

View File

@ -74,6 +74,7 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl
kb.POST("/url", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateKnowledgeFromURL)
kb.POST("/manual", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateManualKnowledge)
kbRead.GET("", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledge)
kbRead.GET("/folders", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledgeFolders)
// Clearing all contents under a KB is a destructive op; gate
// behind Admin instead of Contributor.
kb.With(apiKeyFullAccess()).DELETE("", g.Admin(), g.KBAccessWrite("id"), handler.ClearKnowledgeBaseContents)

View File

@ -84,6 +84,10 @@ type KnowledgeService interface {
page *types.Pagination,
filter types.KnowledgeListFilter,
) (*types.PageResult, error)
// ListKnowledgeFolderTree returns the folder hierarchy derived from the
// folder_path of every knowledge entry in a knowledge base, with per-folder
// document counts. It powers the document sidebar tree.
ListKnowledgeFolderTree(ctx context.Context, kbID string) (*types.KnowledgeFolderTree, error)
// DeleteKnowledge deletes knowledge by ID.
DeleteKnowledge(ctx context.Context, id string) error
// DeleteKnowledgeList deletes multiple knowledge entries by IDs.
@ -241,6 +245,13 @@ type KnowledgeRepository interface {
kbID string,
params *types.KnowledgeCheckParams,
) (bool, *types.Knowledge, error)
// ListKnowledgeFolderCounts aggregates the number of knowledge entries
// stored directly in each folder_path of a knowledge base.
ListKnowledgeFolderCounts(
ctx context.Context,
tenantID uint64,
kbID string,
) ([]*types.KnowledgeFolderCount, error)
// AminusB returns the IDs of knowledge in A that have no counterpart in B,
// comparing file_hash as a multiset (so duplicate-count differences and
// NULL/empty hashes are handled correctly, letting a clone converge).

View File

@ -106,6 +106,13 @@ type KnowledgeListFilter struct {
UpdatedFrom time.Time
// UpdatedTo, when non-zero, keeps rows with updated_at <= UpdatedTo.
UpdatedTo time.Time
// FolderPath is the folder navigated to in the sidebar tree. It is only
// applied when FolderScope is not FolderScopeAny, so the empty string can
// unambiguously mean "the knowledge base root".
FolderPath string
// FolderScope selects whether FolderPath matches exactly or includes
// descendant folders. FolderScopeAny (the default) ignores folders.
FolderScope KnowledgeFolderScope
}
// Knowledge represents a knowledge entity in the system.
@ -144,6 +151,12 @@ type Knowledge struct {
EmbeddingModelID string `json:"embedding_model_id"`
// File name of the knowledge
FileName string `json:"file_name"`
// FolderPath is the canonical relative directory this entry belongs to
// inside the knowledge base, e.g. "docs/spec" for a folder upload of
// "docs/spec/design.md". Empty means the knowledge base root. It is a
// display/navigation concern only: it never affects where the file is
// physically stored (see FilePath).
FolderPath string `json:"folder_path" gorm:"type:varchar(1024);not null;default:''"`
// File type of the knowledge
FileType string `json:"file_type"`
// File size of the knowledge

View File

@ -0,0 +1,213 @@
package types
import (
"sort"
"strings"
)
const (
// MaxKnowledgeFolderDepth caps how many nested levels a folder upload can
// create. Browsers happily hand us arbitrarily deep webkitRelativePath
// values; the cap keeps the sidebar tree renderable and bounds the stored
// path length.
MaxKnowledgeFolderDepth = 16
// MaxKnowledgeFolderPathLength matches the folder_path column width.
MaxKnowledgeFolderPathLength = 1024
// MaxKnowledgeFolderSegmentLength caps a single directory name so one
// pathological segment cannot consume the whole path budget.
MaxKnowledgeFolderSegmentLength = 128
)
// KnowledgeFolderScope selects how KnowledgeListFilter.FolderPath is applied.
type KnowledgeFolderScope string
const (
// FolderScopeAny ignores the folder dimension entirely (flat listing).
FolderScopeAny KnowledgeFolderScope = ""
// FolderScopeExact keeps only entries stored directly in FolderPath.
FolderScopeExact KnowledgeFolderScope = "exact"
// FolderScopeSubtree keeps entries in FolderPath and any descendant folder.
FolderScopeSubtree KnowledgeFolderScope = "subtree"
)
// NormalizeKnowledgeFolderPath turns a client-supplied relative directory into
// the canonical form stored in knowledges.folder_path: forward-slash separated,
// no leading/trailing separator, no "." / ".." segments, depth and length
// capped. The zero value ("") means "knowledge base root".
func NormalizeKnowledgeFolderPath(raw string) string {
if raw == "" {
return ""
}
raw = strings.ReplaceAll(raw, "\\", "/")
segments := make([]string, 0, 8)
for _, segment := range strings.Split(raw, "/") {
segment = strings.TrimSpace(segment)
// Trailing dots and spaces are stripped so "a." and "a" cannot become
// two sibling folders that render identically.
segment = strings.TrimRight(segment, ". ")
if segment == "" || segment == "." || segment == ".." {
continue
}
if len(segment) > MaxKnowledgeFolderSegmentLength {
segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength])
}
if segment == "" {
continue
}
segments = append(segments, segment)
if len(segments) >= MaxKnowledgeFolderDepth {
break
}
}
path := strings.Join(segments, "/")
for len(path) > MaxKnowledgeFolderPathLength && len(segments) > 0 {
segments = segments[:len(segments)-1]
path = strings.Join(segments, "/")
}
return path
}
// SplitKnowledgeRelativePath splits an upload path such as
// "docs/spec/design.md" into its normalized folder path ("docs/spec") and its
// base file name ("design.md"). A plain file name yields an empty folder path,
// so callers can pass the raw `fileName` form field unconditionally.
func SplitKnowledgeRelativePath(raw string) (folderPath string, fileName string) {
normalized := strings.ReplaceAll(raw, "\\", "/")
idx := strings.LastIndex(normalized, "/")
if idx < 0 {
return "", strings.TrimSpace(normalized)
}
fileName = strings.TrimSpace(normalized[idx+1:])
folderPath = NormalizeKnowledgeFolderPath(normalized[:idx])
return folderPath, fileName
}
// KnowledgeFolderName returns the display name of a folder path (its last
// segment). The root folder has no name.
func KnowledgeFolderName(path string) string {
if path == "" {
return ""
}
if idx := strings.LastIndex(path, "/"); idx >= 0 {
return path[idx+1:]
}
return path
}
// KnowledgeFolderParent returns the parent folder path of the given path, or
// an empty string when the path is a root-level folder.
func KnowledgeFolderParent(path string) string {
if idx := strings.LastIndex(path, "/"); idx >= 0 {
return path[:idx]
}
return ""
}
// KnowledgeFolderCount is one row of the folder aggregation query: the number
// of knowledge entries stored directly in a folder path.
type KnowledgeFolderCount struct {
FolderPath string `json:"folder_path" gorm:"column:folder_path"`
Count int64 `json:"count" gorm:"column:count"`
}
// KnowledgeFolderNode is a node of the knowledge base folder tree exposed to
// clients. DocumentCount counts entries stored directly in this folder;
// TotalCount additionally counts every descendant folder's entries.
type KnowledgeFolderNode struct {
// Path is the canonical folder path, e.g. "docs/spec".
Path string `json:"path"`
// Name is the last segment of Path, used as the tree row label.
Name string `json:"name"`
// DocumentCount counts entries whose folder_path equals Path.
DocumentCount int64 `json:"document_count"`
// TotalCount counts entries in Path and in all descendant folders.
TotalCount int64 `json:"total_count"`
// Children are the direct sub-folders, sorted by name.
Children []*KnowledgeFolderNode `json:"children,omitempty"`
}
// KnowledgeFolderTree is the response payload of the folder tree endpoint.
type KnowledgeFolderTree struct {
// RootDocumentCount counts entries that live at the knowledge base root
// (folder_path = ""), i.e. documents that were not uploaded as part of a
// folder.
RootDocumentCount int64 `json:"root_document_count"`
// TotalDocumentCount counts every entry in the knowledge base, matching
// the unfiltered document list total.
TotalDocumentCount int64 `json:"total_document_count"`
// Folders are the top-level folders, sorted by name.
Folders []*KnowledgeFolderNode `json:"folders"`
}
// BuildKnowledgeFolderTree turns the flat per-folder counts returned by the
// repository into a tree. Intermediate folders that hold no documents of their
// own (only sub-folders) are materialized so the hierarchy stays connected.
func BuildKnowledgeFolderTree(counts []*KnowledgeFolderCount) *KnowledgeFolderTree {
tree := &KnowledgeFolderTree{Folders: []*KnowledgeFolderNode{}}
nodes := map[string]*KnowledgeFolderNode{}
children := map[string][]*KnowledgeFolderNode{}
// ensure materializes a folder node and every missing ancestor.
var ensure func(path string) *KnowledgeFolderNode
ensure = func(path string) *KnowledgeFolderNode {
if node, ok := nodes[path]; ok {
return node
}
node := &KnowledgeFolderNode{Path: path, Name: KnowledgeFolderName(path)}
nodes[path] = node
parent := KnowledgeFolderParent(path)
if parent == "" {
tree.Folders = append(tree.Folders, node)
} else {
ensure(parent)
children[parent] = append(children[parent], node)
}
return node
}
for _, row := range counts {
if row == nil {
continue
}
tree.TotalDocumentCount += row.Count
path := NormalizeKnowledgeFolderPath(row.FolderPath)
if path == "" {
tree.RootDocumentCount += row.Count
continue
}
ensure(path).DocumentCount += row.Count
}
// Roll descendant counts up. Deepest paths first so a parent always sees
// final child totals.
paths := make([]string, 0, len(nodes))
for path := range nodes {
paths = append(paths, path)
}
sort.Slice(paths, func(i, j int) bool {
di, dj := strings.Count(paths[i], "/"), strings.Count(paths[j], "/")
if di != dj {
return di > dj
}
return paths[i] < paths[j]
})
for _, path := range paths {
node := nodes[path]
node.TotalCount += node.DocumentCount
if parent := KnowledgeFolderParent(path); parent != "" {
nodes[parent].TotalCount += node.TotalCount
}
}
sortNodes := func(list []*KnowledgeFolderNode) {
sort.Slice(list, func(i, j int) bool {
return strings.ToLower(list[i].Name) < strings.ToLower(list[j].Name)
})
}
for path, list := range children {
sortNodes(list)
nodes[path].Children = list
}
sortNodes(tree.Folders)
return tree
}

View File

@ -0,0 +1,107 @@
package types
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNormalizeKnowledgeFolderPath(t *testing.T) {
cases := []struct {
name string
in string
want string
}{
{"empty stays root", "", ""},
{"plain path", "docs/spec", "docs/spec"},
{"windows separators", `docs\spec`, "docs/spec"},
{"leading and trailing separators", "/docs/spec/", "docs/spec"},
{"collapses empty segments", "docs//spec", "docs/spec"},
{"drops traversal segments", "docs/../../etc/passwd", "docs/etc/passwd"},
{"drops dot segments", "./docs/./spec", "docs/spec"},
{"trims whitespace and trailing dots", " docs . / spec ", "docs/spec"},
{"root only separators", "///", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, NormalizeKnowledgeFolderPath(tc.in))
})
}
}
func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) {
deep := strings.Repeat("a/", MaxKnowledgeFolderDepth+5) + "a"
normalized := NormalizeKnowledgeFolderPath(deep)
assert.Equal(t, MaxKnowledgeFolderDepth, strings.Count(normalized, "/")+1,
"depth must be capped so the sidebar tree stays renderable")
longSegment := strings.Repeat("x", MaxKnowledgeFolderSegmentLength+50)
assert.Len(t, NormalizeKnowledgeFolderPath(longSegment), MaxKnowledgeFolderSegmentLength)
wide := strings.TrimSuffix(strings.Repeat(strings.Repeat("y", 100)+"/", 12), "/")
assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength)
}
func TestSplitKnowledgeRelativePath(t *testing.T) {
cases := []struct {
in string
wantFolder string
wantName string
}{
{"design.md", "", "design.md"},
{"docs/spec/design.md", "docs/spec", "design.md"},
{`docs\spec\design.md`, "docs/spec", "design.md"},
{"/docs/design.md", "docs", "design.md"},
{"docs/../design.md", "docs", "design.md"},
{"docs/", "docs", ""},
}
for _, tc := range cases {
folder, name := SplitKnowledgeRelativePath(tc.in)
assert.Equal(t, tc.wantFolder, folder, "folder for %q", tc.in)
assert.Equal(t, tc.wantName, name, "name for %q", tc.in)
}
}
func TestBuildKnowledgeFolderTree(t *testing.T) {
tree := BuildKnowledgeFolderTree([]*KnowledgeFolderCount{
{FolderPath: "", Count: 2},
{FolderPath: "docs/spec", Count: 3},
{FolderPath: "docs/spec/v2", Count: 1},
{FolderPath: "Assets", Count: 4},
})
assert.Equal(t, int64(2), tree.RootDocumentCount)
assert.Equal(t, int64(10), tree.TotalDocumentCount)
// Top level is sorted case-insensitively: "Assets" before "docs".
require.Len(t, tree.Folders, 2)
assert.Equal(t, "Assets", tree.Folders[0].Path)
assert.Equal(t, int64(4), tree.Folders[0].TotalCount)
docs := tree.Folders[1]
assert.Equal(t, "docs", docs.Path)
// "docs" itself holds no document but is materialized to keep the tree
// connected, and its total rolls up both descendants.
assert.Equal(t, int64(0), docs.DocumentCount)
assert.Equal(t, int64(4), docs.TotalCount)
require.Len(t, docs.Children, 1)
spec := docs.Children[0]
assert.Equal(t, "docs/spec", spec.Path)
assert.Equal(t, "spec", spec.Name)
assert.Equal(t, int64(3), spec.DocumentCount)
assert.Equal(t, int64(4), spec.TotalCount)
require.Len(t, spec.Children, 1)
assert.Equal(t, "docs/spec/v2", spec.Children[0].Path)
assert.Equal(t, int64(1), spec.Children[0].TotalCount)
}
func TestBuildKnowledgeFolderTreeEmpty(t *testing.T) {
tree := BuildKnowledgeFolderTree(nil)
assert.Equal(t, int64(0), tree.TotalDocumentCount)
assert.NotNil(t, tree.Folders, "folders must serialize as [] rather than null")
assert.Empty(t, tree.Folders)
}

View File

@ -0,0 +1,8 @@
-- Fold the relative directory back into file_name before dropping the column.
UPDATE knowledges
SET file_name = folder_path || '/' || file_name
WHERE folder_path <> '' AND file_name <> '';
DROP INDEX IF EXISTS idx_knowledges_folder_path;
ALTER TABLE knowledges DROP COLUMN folder_path;

View File

@ -0,0 +1,23 @@
-- Folder tree support for knowledge bases (SQLite / lite mode).
--
-- Mirrors migrations/versioned/000079_knowledge_folder_path.up.sql.
ALTER TABLE knowledges ADD COLUMN folder_path VARCHAR(1024) NOT NULL DEFAULT '';
-- Backfill legacy folder uploads, which encoded their relative directory in
-- file_name. SQLite has no regex, so the directory prefix is isolated with the
-- rtrim trick: trimming every non-slash character off the right leaves exactly
-- the prefix up to and including the last '/'.
UPDATE knowledges
SET folder_path = substr(
rtrim(file_name, replace(file_name, '/', '')),
1,
length(rtrim(file_name, replace(file_name, '/', ''))) - 1
),
file_name = substr(
file_name,
length(rtrim(file_name, replace(file_name, '/', ''))) + 1
)
WHERE file_name LIKE '%/%';
CREATE INDEX IF NOT EXISTS idx_knowledges_folder_path
ON knowledges (tenant_id, knowledge_base_id, folder_path);

View File

@ -0,0 +1,9 @@
-- Restore the legacy representation: the relative directory is folded back
-- into file_name so a rollback keeps folder uploads distinguishable.
UPDATE knowledges
SET file_name = folder_path || '/' || file_name
WHERE folder_path <> '' AND file_name <> '';
DROP INDEX IF EXISTS idx_knowledges_folder_path;
ALTER TABLE knowledges DROP COLUMN IF EXISTS folder_path;

View File

@ -0,0 +1,21 @@
-- Folder tree support for knowledge bases.
--
-- Folder uploads used to encode their relative directory inside file_name
-- (e.g. "docs/spec/design.md"), which made the document list show the whole
-- path as the title and left no way to query a single folder. The path now
-- lives in its own column and file_name keeps only the base name.
ALTER TABLE knowledges
ADD COLUMN IF NOT EXISTS folder_path VARCHAR(1024) NOT NULL DEFAULT '';
-- Backfill legacy folder uploads so existing knowledge bases render the same
-- tree as newly uploaded folders.
UPDATE knowledges
SET folder_path = LEFT(
REGEXP_REPLACE(file_name, '/[^/]*$', ''),
1024
),
file_name = REGEXP_REPLACE(file_name, '^.*/', '')
WHERE file_name LIKE '%/%';
CREATE INDEX IF NOT EXISTS idx_knowledges_folder_path
ON knowledges (tenant_id, knowledge_base_id, folder_path);