fix(ima): traverse subfolders using list response media IDs

main
yushangcan 2026-09-15 23:40:36 +08:00 committed by lyingbug
parent d3ed78313f
commit fca1cd06f8
4 changed files with 108 additions and 44 deletions

View File

@ -362,34 +362,21 @@ func listAllKBFiles(
return nil, nil, err
}
for _, raw := range resp.KnowledgeList {
// Probe each entry: an entry with a non-empty folder_id is a
// folder; otherwise it's a knowledge item (file / note / etc.).
var probe struct {
FolderID string `json:"folder_id"`
MediaID string `json:"media_id"`
}
_ = json.Unmarshal(raw, &probe)
if probe.FolderID != "" && probe.MediaID == "" {
var fi folderInfo
if err := json.Unmarshal(raw, &fi); err != nil {
continue
}
child := cur.path
if child == "" {
child = fi.Name
} else {
child = cur.path + "/" + fi.Name
}
folderPath[fi.FolderID] = child
stack = append(stack, todo{folderID: fi.FolderID, path: child})
continue
}
if probe.MediaID == "" {
var ki knowledgeInfo
if err := json.Unmarshal(raw, &ki); err != nil || ki.MediaID == "" {
continue // unrecognized shape, skip defensively
}
var ki knowledgeInfo
if err := json.Unmarshal(raw, &ki); err != nil {
if ki.MediaType == mediaTypeFolder {
// List entries identify folders by media_type. Their full
// media_id (including the folder_ prefix) is the recursion key.
child := cur.path
if child == "" {
child = ki.Title
} else {
child = cur.path + "/" + ki.Title
}
folderPath[ki.MediaID] = child
stack = append(stack, todo{folderID: ki.MediaID, path: child})
continue
}
out = append(out, walkedFile{

View File

@ -407,10 +407,11 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
f.setKB("kb1",
[]fakeFile{
{MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"},
{MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"},
{MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_f2", MediaType: mediaTypeMarkdown, Body: "deep"},
},
fakeFolder{FolderID: "f1", Name: "Outer"},
fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"},
fakeFolder{FolderID: "folder_f1", Name: "Outer"},
fakeFolder{FolderID: "folder_f2", Name: "Inner", ParentFolderID: "folder_f1"},
fakeFolder{FolderID: "folder_empty", Name: "Empty"},
)
items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
@ -418,7 +419,13 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
t.Fatalf("FetchAll: %v", err)
}
deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep"))
if len(items) != 2 {
t.Fatalf("expected only the root and nested files, got %s", describeItems(items))
}
deep := mustFindItem(t, items, logicalKey("kb1", "folder_f2", "Deep"))
if string(deep.Content) != "deep" {
t.Errorf("nested content = %q, want deep", deep.Content)
}
if deep.Metadata["folder_path"] != "Outer/Inner" {
t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
}
@ -426,6 +433,74 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
if root.Metadata["folder_path"] != "" {
t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"])
}
for _, id := range []string{"folder_f1", "folder_f2", "folder_empty"} {
if got := f.callCount("get_knowledge_list:" + id); got != 1 {
t.Errorf("list folder %q calls = %d, want 1 (with the folder_ prefix preserved)", id, got)
}
if got := f.callCount("get_media_info:" + id); got != 0 {
t.Errorf("folder %q was fetched as a file %d times", id, got)
}
}
}
func TestFetchIncremental_NestedFolders(t *testing.T) {
f := newFakeIMA(t)
folders := []fakeFolder{
{FolderID: "folder_outer", Name: "Outer"},
{FolderID: "folder_inner", Name: "Inner", ParentFolderID: "folder_outer"},
}
file := fakeFile{
MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_inner",
MediaType: mediaTypeMarkdown, Body: "deep",
}
f.setKB("kb1", []fakeFile{file}, folders...)
c := NewConnector()
cfg := f.config("kb1")
key := logicalKey("kb1", "folder_inner", "Deep")
items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
if err != nil {
t.Fatalf("first FetchIncremental: %v", err)
}
deep := mustFindItem(t, items, key)
if deep.Metadata["folder_path"] != "Outer/Inner" {
t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
}
state := decodeCursor(t, cursor).KBLogical["kb1"]
if len(state) != 1 || state[key] != file.MediaID {
t.Fatalf("cursor should contain only the nested file, got %#v", state)
}
items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)
if err != nil {
t.Fatalf("unchanged FetchIncremental: %v", err)
}
if len(items) != 0 {
t.Fatalf("unchanged nested file must not be emitted or tombstoned, got %s", describeItems(items))
}
if got := f.callCount("get_media_info:" + file.MediaID); got != 1 {
t.Errorf("nested file fetched %d times, want 1", got)
}
if decodeCursor(t, cursor).KBLogical["kb1"][key] != file.MediaID {
t.Fatal("unchanged nested file must remain in the cursor")
}
f.setKB("kb1", nil, folders...)
items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)
if err != nil {
t.Fatalf("deleted FetchIncremental: %v", err)
}
if len(items) != 1 || !mustFindItem(t, items, key).IsDeleted {
t.Fatalf("removed nested file must be tombstoned, got %s", describeItems(items))
}
if state := decodeCursor(t, cursor).KBLogical["kb1"]; len(state) != 0 {
t.Errorf("empty folders must not be recorded as files, got %#v", state)
}
for _, folder := range folders {
if got := f.callCount("get_media_info:" + folder.FolderID); got != 0 {
t.Errorf("folder %q was fetched as a file %d times", folder.FolderID, got)
}
}
}
func TestListResources_ReturnsAddableBases(t *testing.T) {

View File

@ -30,7 +30,7 @@ type fakeFile struct {
Title string
ParentFolderID string
// MediaType is what get_media_info reports for this file.
// MediaType is reported by both get_knowledge_list and get_media_info.
MediaType int32
// Body is what the download URL serves.
Body string
@ -190,6 +190,7 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
case "get_knowledge_list":
kbID, folderID := str("knowledge_base_id"), str("folder_id")
f.record("get_knowledge_list:" + folderID)
f.mu.Lock()
files := f.files[kbID][folderID]
folders := f.folders[kbID][folderID]
@ -197,19 +198,21 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
var list []json.RawMessage
for _, folder := range folders {
b, _ := json.Marshal(folderInfo{
FolderID: folder.FolderID,
Name: folder.Name,
ParentFolderID: folder.ParentFolderID,
// Folder entries use the same fields as files in list responses.
b, _ := json.Marshal(map[string]interface{}{
"media_id": folder.FolderID,
"title": folder.Name,
"parent_folder_id": folder.ParentFolderID,
"media_type": 99,
})
list = append(list, b)
}
for _, file := range files {
// IMA omits media_type in list responses.
b, _ := json.Marshal(map[string]interface{}{
"media_id": file.MediaID,
"title": file.Title,
"parent_folder_id": file.ParentFolderID,
"media_type": file.MediaType,
})
list = append(list, b)
}

View File

@ -156,10 +156,9 @@ type getKnowledgeBaseResp struct {
Infos map[string]knowledgeBaseInfo `json:"infos"`
}
// knowledgeInfo mirrors IMA's KnowledgeInfo (files, notes, etc.).
// IMPORTANT: IMA does not expose media_type in the list response, we must call
// get_media_info to discover it. However, folders are distinguished by presence
// of a non-empty folder_id — see folderInfo below.
// knowledgeInfo mirrors entries in IMA's knowledge_list (files, notes, folders).
// Folders have media_type=99 and use media_id as the folder ID for child listings.
// File download details are resolved separately through get_media_info.
type knowledgeInfo struct {
MediaID string `json:"media_id"`
Title string `json:"title"`
@ -167,7 +166,7 @@ type knowledgeInfo struct {
MediaType int32 `json:"media_type,omitempty"`
}
// folderInfo mirrors IMA's FolderInfo.
// folderInfo mirrors IMA's FolderInfo in current_path, not knowledge_list.
type folderInfo struct {
FolderID string `json:"folder_id"`
Name string `json:"name"`
@ -202,9 +201,8 @@ func (f folderInfo) FolderCount() int64 {
}
// getKnowledgeListResp — get_knowledge_list response.
// According to `references/api.md`, the response returns knowledge_list mixed
// with folders; we decode a loose JSON view so entries without a folder_id are
// treated as knowledge and those with folder_id are treated as folders.
// The response returns files and folders together in knowledge_list;
// media_type distinguishes folders from knowledge items.
type getKnowledgeListResp struct {
// KnowledgeList is a loose slice of raw JSON objects because IMA returns
// files and folders in the same array. We iterate and inspect fields.
@ -309,6 +307,7 @@ const (
mediaTypeVideo int32 = 16
mediaTypeHTML int32 = 20
mediaTypeEPUB int32 = 21
mediaTypeFolder int32 = 99
)
// extensionForMediaType maps IMA media types to a file extension.