From fca1cd06f818f65d7776114ac2a697315af17efd Mon Sep 17 00:00:00 2001 From: yushangcan <2551520580@qq.com> Date: Tue, 15 Sep 2026 23:40:36 +0800 Subject: [PATCH] fix(ima): traverse subfolders using list response media IDs --- .../datasource/connector/ima/connector.go | 39 +++------ .../connector/ima/connector_test.go | 83 ++++++++++++++++++- .../datasource/connector/ima/fake_ima_test.go | 15 ++-- internal/datasource/connector/ima/types.go | 15 ++-- 4 files changed, 108 insertions(+), 44 deletions(-) diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.go index 6111ffc6..4bf3ba17 100644 --- a/internal/datasource/connector/ima/connector.go +++ b/internal/datasource/connector/ima/connector.go @@ -362,34 +362,21 @@ func listAllKBFiles( return nil, nil, err } for _, raw := range resp.KnowledgeList { - // Probe each entry: an entry with a non-empty folder_id is a - // folder; otherwise it's a knowledge item (file / note / etc.). - var probe struct { - FolderID string `json:"folder_id"` - MediaID string `json:"media_id"` - } - _ = json.Unmarshal(raw, &probe) - - if probe.FolderID != "" && probe.MediaID == "" { - var fi folderInfo - if err := json.Unmarshal(raw, &fi); err != nil { - continue - } - child := cur.path - if child == "" { - child = fi.Name - } else { - child = cur.path + "/" + fi.Name - } - folderPath[fi.FolderID] = child - stack = append(stack, todo{folderID: fi.FolderID, path: child}) - continue - } - if probe.MediaID == "" { + var ki knowledgeInfo + if err := json.Unmarshal(raw, &ki); err != nil || ki.MediaID == "" { continue // unrecognized shape, skip defensively } - var ki knowledgeInfo - if err := json.Unmarshal(raw, &ki); err != nil { + if ki.MediaType == mediaTypeFolder { + // List entries identify folders by media_type. Their full + // media_id (including the folder_ prefix) is the recursion key. + child := cur.path + if child == "" { + child = ki.Title + } else { + child = cur.path + "/" + ki.Title + } + folderPath[ki.MediaID] = child + stack = append(stack, todo{folderID: ki.MediaID, path: child}) continue } out = append(out, walkedFile{ diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.go index 42970c8c..7a99f790 100644 --- a/internal/datasource/connector/ima/connector_test.go +++ b/internal/datasource/connector/ima/connector_test.go @@ -407,10 +407,11 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) { f.setKB("kb1", []fakeFile{ {MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"}, - {MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"}, + {MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_f2", MediaType: mediaTypeMarkdown, Body: "deep"}, }, - fakeFolder{FolderID: "f1", Name: "Outer"}, - fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"}, + fakeFolder{FolderID: "folder_f1", Name: "Outer"}, + fakeFolder{FolderID: "folder_f2", Name: "Inner", ParentFolderID: "folder_f1"}, + fakeFolder{FolderID: "folder_empty", Name: "Empty"}, ) items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) @@ -418,7 +419,13 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) { t.Fatalf("FetchAll: %v", err) } - deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep")) + if len(items) != 2 { + t.Fatalf("expected only the root and nested files, got %s", describeItems(items)) + } + deep := mustFindItem(t, items, logicalKey("kb1", "folder_f2", "Deep")) + if string(deep.Content) != "deep" { + t.Errorf("nested content = %q, want deep", deep.Content) + } if deep.Metadata["folder_path"] != "Outer/Inner" { t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"]) } @@ -426,6 +433,74 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) { if root.Metadata["folder_path"] != "" { t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"]) } + for _, id := range []string{"folder_f1", "folder_f2", "folder_empty"} { + if got := f.callCount("get_knowledge_list:" + id); got != 1 { + t.Errorf("list folder %q calls = %d, want 1 (with the folder_ prefix preserved)", id, got) + } + if got := f.callCount("get_media_info:" + id); got != 0 { + t.Errorf("folder %q was fetched as a file %d times", id, got) + } + } +} + +func TestFetchIncremental_NestedFolders(t *testing.T) { + f := newFakeIMA(t) + folders := []fakeFolder{ + {FolderID: "folder_outer", Name: "Outer"}, + {FolderID: "folder_inner", Name: "Inner", ParentFolderID: "folder_outer"}, + } + file := fakeFile{ + MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_inner", + MediaType: mediaTypeMarkdown, Body: "deep", + } + f.setKB("kb1", []fakeFile{file}, folders...) + c := NewConnector() + cfg := f.config("kb1") + key := logicalKey("kb1", "folder_inner", "Deep") + + items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + deep := mustFindItem(t, items, key) + if deep.Metadata["folder_path"] != "Outer/Inner" { + t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"]) + } + state := decodeCursor(t, cursor).KBLogical["kb1"] + if len(state) != 1 || state[key] != file.MediaID { + t.Fatalf("cursor should contain only the nested file, got %#v", state) + } + + items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("unchanged FetchIncremental: %v", err) + } + if len(items) != 0 { + t.Fatalf("unchanged nested file must not be emitted or tombstoned, got %s", describeItems(items)) + } + if got := f.callCount("get_media_info:" + file.MediaID); got != 1 { + t.Errorf("nested file fetched %d times, want 1", got) + } + if decodeCursor(t, cursor).KBLogical["kb1"][key] != file.MediaID { + t.Fatal("unchanged nested file must remain in the cursor") + } + + f.setKB("kb1", nil, folders...) + items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("deleted FetchIncremental: %v", err) + } + if len(items) != 1 || !mustFindItem(t, items, key).IsDeleted { + t.Fatalf("removed nested file must be tombstoned, got %s", describeItems(items)) + } + if state := decodeCursor(t, cursor).KBLogical["kb1"]; len(state) != 0 { + t.Errorf("empty folders must not be recorded as files, got %#v", state) + } + for _, folder := range folders { + if got := f.callCount("get_media_info:" + folder.FolderID); got != 0 { + t.Errorf("folder %q was fetched as a file %d times", folder.FolderID, got) + } + } } func TestListResources_ReturnsAddableBases(t *testing.T) { diff --git a/internal/datasource/connector/ima/fake_ima_test.go b/internal/datasource/connector/ima/fake_ima_test.go index 1b503ab6..36894551 100644 --- a/internal/datasource/connector/ima/fake_ima_test.go +++ b/internal/datasource/connector/ima/fake_ima_test.go @@ -30,7 +30,7 @@ type fakeFile struct { Title string ParentFolderID string - // MediaType is what get_media_info reports for this file. + // MediaType is reported by both get_knowledge_list and get_media_info. MediaType int32 // Body is what the download URL serves. Body string @@ -190,6 +190,7 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) { case "get_knowledge_list": kbID, folderID := str("knowledge_base_id"), str("folder_id") + f.record("get_knowledge_list:" + folderID) f.mu.Lock() files := f.files[kbID][folderID] folders := f.folders[kbID][folderID] @@ -197,19 +198,21 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) { var list []json.RawMessage for _, folder := range folders { - b, _ := json.Marshal(folderInfo{ - FolderID: folder.FolderID, - Name: folder.Name, - ParentFolderID: folder.ParentFolderID, + // Folder entries use the same fields as files in list responses. + b, _ := json.Marshal(map[string]interface{}{ + "media_id": folder.FolderID, + "title": folder.Name, + "parent_folder_id": folder.ParentFolderID, + "media_type": 99, }) list = append(list, b) } for _, file := range files { - // IMA omits media_type in list responses. b, _ := json.Marshal(map[string]interface{}{ "media_id": file.MediaID, "title": file.Title, "parent_folder_id": file.ParentFolderID, + "media_type": file.MediaType, }) list = append(list, b) } diff --git a/internal/datasource/connector/ima/types.go b/internal/datasource/connector/ima/types.go index 48569f2e..3e4fadbc 100644 --- a/internal/datasource/connector/ima/types.go +++ b/internal/datasource/connector/ima/types.go @@ -156,10 +156,9 @@ type getKnowledgeBaseResp struct { Infos map[string]knowledgeBaseInfo `json:"infos"` } -// knowledgeInfo mirrors IMA's KnowledgeInfo (files, notes, etc.). -// IMPORTANT: IMA does not expose media_type in the list response, we must call -// get_media_info to discover it. However, folders are distinguished by presence -// of a non-empty folder_id — see folderInfo below. +// knowledgeInfo mirrors entries in IMA's knowledge_list (files, notes, folders). +// Folders have media_type=99 and use media_id as the folder ID for child listings. +// File download details are resolved separately through get_media_info. type knowledgeInfo struct { MediaID string `json:"media_id"` Title string `json:"title"` @@ -167,7 +166,7 @@ type knowledgeInfo struct { MediaType int32 `json:"media_type,omitempty"` } -// folderInfo mirrors IMA's FolderInfo. +// folderInfo mirrors IMA's FolderInfo in current_path, not knowledge_list. type folderInfo struct { FolderID string `json:"folder_id"` Name string `json:"name"` @@ -202,9 +201,8 @@ func (f folderInfo) FolderCount() int64 { } // getKnowledgeListResp — get_knowledge_list response. -// According to `references/api.md`, the response returns knowledge_list mixed -// with folders; we decode a loose JSON view so entries without a folder_id are -// treated as knowledge and those with folder_id are treated as folders. +// The response returns files and folders together in knowledge_list; +// media_type distinguishes folders from knowledge items. type getKnowledgeListResp struct { // KnowledgeList is a loose slice of raw JSON objects because IMA returns // files and folders in the same array. We iterate and inspect fields. @@ -309,6 +307,7 @@ const ( mediaTypeVideo int32 = 16 mediaTypeHTML int32 = 20 mediaTypeEPUB int32 = 21 + mediaTypeFolder int32 = 99 ) // extensionForMediaType maps IMA media types to a file extension.