fix(ima): traverse subfolders using list response media IDs
parent
d3ed78313f
commit
fca1cd06f8
|
|
@ -362,34 +362,21 @@ func listAllKBFiles(
|
|||
return nil, nil, err
|
||||
}
|
||||
for _, raw := range resp.KnowledgeList {
|
||||
// Probe each entry: an entry with a non-empty folder_id is a
|
||||
// folder; otherwise it's a knowledge item (file / note / etc.).
|
||||
var probe struct {
|
||||
FolderID string `json:"folder_id"`
|
||||
MediaID string `json:"media_id"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &probe)
|
||||
|
||||
if probe.FolderID != "" && probe.MediaID == "" {
|
||||
var fi folderInfo
|
||||
if err := json.Unmarshal(raw, &fi); err != nil {
|
||||
continue
|
||||
}
|
||||
child := cur.path
|
||||
if child == "" {
|
||||
child = fi.Name
|
||||
} else {
|
||||
child = cur.path + "/" + fi.Name
|
||||
}
|
||||
folderPath[fi.FolderID] = child
|
||||
stack = append(stack, todo{folderID: fi.FolderID, path: child})
|
||||
continue
|
||||
}
|
||||
if probe.MediaID == "" {
|
||||
var ki knowledgeInfo
|
||||
if err := json.Unmarshal(raw, &ki); err != nil || ki.MediaID == "" {
|
||||
continue // unrecognized shape, skip defensively
|
||||
}
|
||||
var ki knowledgeInfo
|
||||
if err := json.Unmarshal(raw, &ki); err != nil {
|
||||
if ki.MediaType == mediaTypeFolder {
|
||||
// List entries identify folders by media_type. Their full
|
||||
// media_id (including the folder_ prefix) is the recursion key.
|
||||
child := cur.path
|
||||
if child == "" {
|
||||
child = ki.Title
|
||||
} else {
|
||||
child = cur.path + "/" + ki.Title
|
||||
}
|
||||
folderPath[ki.MediaID] = child
|
||||
stack = append(stack, todo{folderID: ki.MediaID, path: child})
|
||||
continue
|
||||
}
|
||||
out = append(out, walkedFile{
|
||||
|
|
|
|||
|
|
@ -407,10 +407,11 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
|
|||
f.setKB("kb1",
|
||||
[]fakeFile{
|
||||
{MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"},
|
||||
{MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"},
|
||||
{MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_f2", MediaType: mediaTypeMarkdown, Body: "deep"},
|
||||
},
|
||||
fakeFolder{FolderID: "f1", Name: "Outer"},
|
||||
fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"},
|
||||
fakeFolder{FolderID: "folder_f1", Name: "Outer"},
|
||||
fakeFolder{FolderID: "folder_f2", Name: "Inner", ParentFolderID: "folder_f1"},
|
||||
fakeFolder{FolderID: "folder_empty", Name: "Empty"},
|
||||
)
|
||||
|
||||
items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
|
||||
|
|
@ -418,7 +419,13 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
|
|||
t.Fatalf("FetchAll: %v", err)
|
||||
}
|
||||
|
||||
deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep"))
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("expected only the root and nested files, got %s", describeItems(items))
|
||||
}
|
||||
deep := mustFindItem(t, items, logicalKey("kb1", "folder_f2", "Deep"))
|
||||
if string(deep.Content) != "deep" {
|
||||
t.Errorf("nested content = %q, want deep", deep.Content)
|
||||
}
|
||||
if deep.Metadata["folder_path"] != "Outer/Inner" {
|
||||
t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
|
||||
}
|
||||
|
|
@ -426,6 +433,74 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
|
|||
if root.Metadata["folder_path"] != "" {
|
||||
t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"])
|
||||
}
|
||||
for _, id := range []string{"folder_f1", "folder_f2", "folder_empty"} {
|
||||
if got := f.callCount("get_knowledge_list:" + id); got != 1 {
|
||||
t.Errorf("list folder %q calls = %d, want 1 (with the folder_ prefix preserved)", id, got)
|
||||
}
|
||||
if got := f.callCount("get_media_info:" + id); got != 0 {
|
||||
t.Errorf("folder %q was fetched as a file %d times", id, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchIncremental_NestedFolders(t *testing.T) {
|
||||
f := newFakeIMA(t)
|
||||
folders := []fakeFolder{
|
||||
{FolderID: "folder_outer", Name: "Outer"},
|
||||
{FolderID: "folder_inner", Name: "Inner", ParentFolderID: "folder_outer"},
|
||||
}
|
||||
file := fakeFile{
|
||||
MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_inner",
|
||||
MediaType: mediaTypeMarkdown, Body: "deep",
|
||||
}
|
||||
f.setKB("kb1", []fakeFile{file}, folders...)
|
||||
c := NewConnector()
|
||||
cfg := f.config("kb1")
|
||||
key := logicalKey("kb1", "folder_inner", "Deep")
|
||||
|
||||
items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first FetchIncremental: %v", err)
|
||||
}
|
||||
deep := mustFindItem(t, items, key)
|
||||
if deep.Metadata["folder_path"] != "Outer/Inner" {
|
||||
t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
|
||||
}
|
||||
state := decodeCursor(t, cursor).KBLogical["kb1"]
|
||||
if len(state) != 1 || state[key] != file.MediaID {
|
||||
t.Fatalf("cursor should contain only the nested file, got %#v", state)
|
||||
}
|
||||
|
||||
items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)
|
||||
if err != nil {
|
||||
t.Fatalf("unchanged FetchIncremental: %v", err)
|
||||
}
|
||||
if len(items) != 0 {
|
||||
t.Fatalf("unchanged nested file must not be emitted or tombstoned, got %s", describeItems(items))
|
||||
}
|
||||
if got := f.callCount("get_media_info:" + file.MediaID); got != 1 {
|
||||
t.Errorf("nested file fetched %d times, want 1", got)
|
||||
}
|
||||
if decodeCursor(t, cursor).KBLogical["kb1"][key] != file.MediaID {
|
||||
t.Fatal("unchanged nested file must remain in the cursor")
|
||||
}
|
||||
|
||||
f.setKB("kb1", nil, folders...)
|
||||
items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)
|
||||
if err != nil {
|
||||
t.Fatalf("deleted FetchIncremental: %v", err)
|
||||
}
|
||||
if len(items) != 1 || !mustFindItem(t, items, key).IsDeleted {
|
||||
t.Fatalf("removed nested file must be tombstoned, got %s", describeItems(items))
|
||||
}
|
||||
if state := decodeCursor(t, cursor).KBLogical["kb1"]; len(state) != 0 {
|
||||
t.Errorf("empty folders must not be recorded as files, got %#v", state)
|
||||
}
|
||||
for _, folder := range folders {
|
||||
if got := f.callCount("get_media_info:" + folder.FolderID); got != 0 {
|
||||
t.Errorf("folder %q was fetched as a file %d times", folder.FolderID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListResources_ReturnsAddableBases(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ type fakeFile struct {
|
|||
Title string
|
||||
ParentFolderID string
|
||||
|
||||
// MediaType is what get_media_info reports for this file.
|
||||
// MediaType is reported by both get_knowledge_list and get_media_info.
|
||||
MediaType int32
|
||||
// Body is what the download URL serves.
|
||||
Body string
|
||||
|
|
@ -190,6 +190,7 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
case "get_knowledge_list":
|
||||
kbID, folderID := str("knowledge_base_id"), str("folder_id")
|
||||
f.record("get_knowledge_list:" + folderID)
|
||||
f.mu.Lock()
|
||||
files := f.files[kbID][folderID]
|
||||
folders := f.folders[kbID][folderID]
|
||||
|
|
@ -197,19 +198,21 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
var list []json.RawMessage
|
||||
for _, folder := range folders {
|
||||
b, _ := json.Marshal(folderInfo{
|
||||
FolderID: folder.FolderID,
|
||||
Name: folder.Name,
|
||||
ParentFolderID: folder.ParentFolderID,
|
||||
// Folder entries use the same fields as files in list responses.
|
||||
b, _ := json.Marshal(map[string]interface{}{
|
||||
"media_id": folder.FolderID,
|
||||
"title": folder.Name,
|
||||
"parent_folder_id": folder.ParentFolderID,
|
||||
"media_type": 99,
|
||||
})
|
||||
list = append(list, b)
|
||||
}
|
||||
for _, file := range files {
|
||||
// IMA omits media_type in list responses.
|
||||
b, _ := json.Marshal(map[string]interface{}{
|
||||
"media_id": file.MediaID,
|
||||
"title": file.Title,
|
||||
"parent_folder_id": file.ParentFolderID,
|
||||
"media_type": file.MediaType,
|
||||
})
|
||||
list = append(list, b)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -156,10 +156,9 @@ type getKnowledgeBaseResp struct {
|
|||
Infos map[string]knowledgeBaseInfo `json:"infos"`
|
||||
}
|
||||
|
||||
// knowledgeInfo mirrors IMA's KnowledgeInfo (files, notes, etc.).
|
||||
// IMPORTANT: IMA does not expose media_type in the list response, we must call
|
||||
// get_media_info to discover it. However, folders are distinguished by presence
|
||||
// of a non-empty folder_id — see folderInfo below.
|
||||
// knowledgeInfo mirrors entries in IMA's knowledge_list (files, notes, folders).
|
||||
// Folders have media_type=99 and use media_id as the folder ID for child listings.
|
||||
// File download details are resolved separately through get_media_info.
|
||||
type knowledgeInfo struct {
|
||||
MediaID string `json:"media_id"`
|
||||
Title string `json:"title"`
|
||||
|
|
@ -167,7 +166,7 @@ type knowledgeInfo struct {
|
|||
MediaType int32 `json:"media_type,omitempty"`
|
||||
}
|
||||
|
||||
// folderInfo mirrors IMA's FolderInfo.
|
||||
// folderInfo mirrors IMA's FolderInfo in current_path, not knowledge_list.
|
||||
type folderInfo struct {
|
||||
FolderID string `json:"folder_id"`
|
||||
Name string `json:"name"`
|
||||
|
|
@ -202,9 +201,8 @@ func (f folderInfo) FolderCount() int64 {
|
|||
}
|
||||
|
||||
// getKnowledgeListResp — get_knowledge_list response.
|
||||
// According to `references/api.md`, the response returns knowledge_list mixed
|
||||
// with folders; we decode a loose JSON view so entries without a folder_id are
|
||||
// treated as knowledge and those with folder_id are treated as folders.
|
||||
// The response returns files and folders together in knowledge_list;
|
||||
// media_type distinguishes folders from knowledge items.
|
||||
type getKnowledgeListResp struct {
|
||||
// KnowledgeList is a loose slice of raw JSON objects because IMA returns
|
||||
// files and folders in the same array. We iterate and inspect fields.
|
||||
|
|
@ -309,6 +307,7 @@ const (
|
|||
mediaTypeVideo int32 = 16
|
||||
mediaTypeHTML int32 = 20
|
||||
mediaTypeEPUB int32 = 21
|
||||
mediaTypeFolder int32 = 99
|
||||
)
|
||||
|
||||
// extensionForMediaType maps IMA media types to a file extension.
|
||||
|
|
|
|||
Loading…
Reference in New Issue