diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 99bdea59..205b3490 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -5486,7 +5486,7 @@ export default { lark_drive: 'Sync documents, spreadsheets and files from a Lark Drive folder (Feishu international)', notion: 'Sync pages and databases from Notion', yuque: 'Sync documents from Yuque knowledge bases', - ima: 'Sync documents, notes and files from Tencent IMA knowledge bases', + ima: 'Sync documents and files from Tencent IMA knowledge bases (notes and AI sessions are not supported yet)', rss: 'Sync articles from RSS / Atom feeds' }, drive: { diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 796cbf9f..32116e89 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -704,7 +704,7 @@ export default { lark_drive: "Lark 드라이브 폴더에서 문서, 스프레드시트, 파일 동기화", notion: 'Notion에서 페이지 및 데이터베이스 동기화', yuque: '위큐 지식베이스에서 문서 동기화', - ima: 'Tencent IMA 지식베이스에서 문서, 노트 및 파일 동기화', + ima: 'Tencent IMA 지식베이스에서 문서 및 파일 동기화 (노트와 AI 세션은 아직 지원되지 않음)', rss: 'RSS / Atom 피드에서 글 동기화' }, connector: { diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index a19624b8..7ac69061 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -704,7 +704,7 @@ export default { lark_drive: 'Синхронизация документов, таблиц и файлов из папки Lark Drive', notion: 'Синхронизация страниц и баз данных из Notion', yuque: 'Синхронизация документов из баз знаний Yuque', - ima: 'Синхронизация документов, заметок и файлов из баз знаний Tencent IMA', + ima: 'Синхронизация документов и файлов из баз знаний Tencent IMA (заметки и ИИ-сессии пока не поддерживаются)', rss: 'Синхронизация статей из лент RSS / Atom' }, connector: { diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index a6bc1778..363c0d5f 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -704,7 +704,7 @@ export default { lark_drive: "同步 Lark 云盘文件夹中的文档、表格、文件(飞书国际版)", notion: '同步 Notion 中的页面和数据库', yuque: '同步语雀知识库中的文档', - ima: '同步腾讯 IMA 知识库中的文档、笔记与文件', + ima: '同步腾讯 IMA 知识库中的文档与文件(暂不支持笔记与 AI 会话)', rss: '同步 RSS / Atom 订阅源中的文章' }, connector: { diff --git a/internal/datasource/connector/ima/client.go b/internal/datasource/connector/ima/client.go index 95a8395e..aef65cda 100644 --- a/internal/datasource/connector/ima/client.go +++ b/internal/datasource/connector/ima/client.go @@ -1,3 +1,6 @@ +// Package ima implements the WeKnora data source connector for Tencent IMA +// (ima.qq.com), syncing documents and files out of IMA knowledge bases via the +// /openapi/wiki/v1 OpenAPI. package ima import ( @@ -12,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/utils" ) const ( @@ -112,7 +116,7 @@ func (c *client) callAPI(ctx context.Context, action string, req interface{}, re } respBody, readErr := io.ReadAll(resp.Body) - resp.Body.Close() + _ = resp.Body.Close() if readErr != nil { lastErr = fmt.Errorf("read response: %w", readErr) if attempt < maxRetries { @@ -124,9 +128,11 @@ func (c *client) callAPI(ctx context.Context, action string, req interface{}, re return lastErr } + // The body is only ever logged on an error path: successful responses + // (get_media_info in particular) carry pre-signed storage URLs and + // per-request auth headers that must not land in the server log. bodyPreview := truncate(string(respBody), 500) - logger.Infof(ctx, "[IMA] POST %s → status=%d bodyLen=%d body=%s", - action, resp.StatusCode, len(respBody), bodyPreview) + logger.Infof(ctx, "[IMA] POST %s → status=%d bodyLen=%d", action, resp.StatusCode, len(respBody)) // HTTP-level auth failures: surface as ErrInvalidCredentials. if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { @@ -167,9 +173,11 @@ func (c *client) callAPI(ctx context.Context, action string, req interface{}, re } // Business-level errors. - // 110030 (无权限) / 110001 (参数非法) with common auth hint patterns are - // mapped to ErrInvalidCredentials so the service marks the source in - // `error` state and stops scheduling until re-authenticated. + // 110030 (无权限) is mapped to ErrInvalidCredentials so the service marks + // the source in `error` state and stops scheduling until re-authenticated. + // 110021 (限频) is retried. Everything else — including 110001 (参数非法), + // which is a bug on our side rather than a credential problem — surfaces + // verbatim so the user sees IMA's own message. if env.Code != 0 { if env.Code == 110030 { return fmt.Errorf("%w: ima code=%d msg=%s", @@ -294,6 +302,13 @@ func (c *client) DownloadURL(ctx context.Context, u urlInfo) ([]byte, string, er if u.URL == "" { return nil, "", fmt.Errorf("empty url") } + // The URL comes from an API response, so it is attacker-influenced as far + // as this process is concerned. downloadClient already guards the dial and + // every redirect hop; this rejects an unsafe target before the first + // connection is even attempted. + if err := utils.ValidateURLForSSRF(u.URL); err != nil { + return nil, "", fmt.Errorf("media URL rejected: %w", err) + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.URL, nil) if err != nil { return nil, "", fmt.Errorf("create download request: %w", err) @@ -307,7 +322,7 @@ func (c *client) DownloadURL(ctx context.Context, u urlInfo) ([]byte, string, er if err != nil { return nil, "", fmt.Errorf("download: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, "", fmt.Errorf("download http error: status=%d", resp.StatusCode) diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.go index b21a9c82..dd579544 100644 --- a/internal/datasource/connector/ima/connector.go +++ b/internal/datasource/connector/ima/connector.go @@ -42,10 +42,11 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig return nil } -// ResolveResourceAncestors: IMA knowledge bases are exposed as a flat list of -// top-level resources, so a lazy-loaded picker has no ancestors to reveal. +// ResolveResourceAncestors returns nothing: IMA knowledge bases are exposed as +// a flat list of top-level resources, so a lazy-loaded picker has no ancestors +// to reveal. func (c *Connector) ResolveResourceAncestors( - ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, + _ context.Context, _ *types.DataSourceConfig, _ []string, ) ([]string, error) { return []string{}, nil } @@ -103,7 +104,7 @@ func (c *Connector) ListResources( return nil, fmt.Errorf("search_knowledge_base fallback: %w", err) } for _, b := range resp.InfoList { - bases = append(bases, kbLite{ID: b.ID, Name: b.Name, CoverURL: b.CoverURL}) + bases = append(bases, kbLite(b)) } if resp.IsEnd || resp.NextCursor == "" { break @@ -237,7 +238,6 @@ func (c *Connector) walk( newCursor := &imaCursor{ LastSyncTime: time.Now(), KBLogical: make(map[string]map[string]string), - KBMedia: make(map[string]map[string]string), } var out []types.FetchedItem @@ -247,24 +247,22 @@ func (c *Connector) walk( return nil, nil, fmt.Errorf("list KB %s: %w", kbID, err) } - currentLogical := make(map[string]string, len(files)) - currentMedia := make(map[string]string, len(files)) + // present holds every logical_key IMA listed this cycle; it drives + // delete detection. syncedLogical holds only the keys we actually + // resolved this cycle and is what gets persisted as the cursor, so a + // transient failure is retried next run rather than being remembered + // as "unchanged" forever. + present := make(map[string]struct{}, len(files)) + syncedLogical := make(map[string]string, len(files)) keys := make([]string, len(files)) for i, f := range files { - // f.MediaType is 0 in list responses (IMA omits it there); we rely on - // (kb_id, parent_folder_id, title) which IMA enforces uniqueness on - // via check_repeated_names. Encoding media_type as 0 is fine as long - // as it stays consistent across syncs — which it does, since 0 is a - // constant here. - key := logicalKey(kbID, f.ParentFolderID, f.Title, f.MediaType) + key := logicalKey(kbID, f.ParentFolderID, f.Title) keys[i] = key - currentLogical[key] = f.MediaID - currentMedia[f.MediaID] = f.Title + present[key] = struct{}{} } - newCursor.KBLogical[kbID] = currentLogical - newCursor.KBMedia[kbID] = currentMedia + newCursor.KBLogical[kbID] = syncedLogical - replaced := 0 + var replaced, failed int for i, f := range files { key := keys[i] @@ -273,28 +271,41 @@ func (c *Connector) walk( if prevSet, ok := prev.KBLogical[kbID]; ok { if prevMediaID, seen := prevSet[key]; seen { if prevMediaID == f.MediaID { - continue // unchanged + // Unchanged: carry the entry forward so it is not + // re-fetched next cycle either. + syncedLogical[key] = f.MediaID + continue } replaced++ // media_id changed → replacement, fall through to re-fetch } } } - item, ok := fetchOneMedia(ctx, cli, cfg, kbID, key, f, folderPath) - if !ok { - continue + item, outcome := fetchOneMedia(ctx, cli, kbID, key, f, folderPath) + switch outcome { + case fetchOK: + syncedLogical[key] = f.MediaID + out = append(out, item) + case fetchSkipped: + // Deterministic skip (unsupported media type, no URL): record it + // so we do not pay for get_media_info on every future sync. + syncedLogical[key] = f.MediaID + case fetchFailed: + // Transient: leave the key out of the cursor so the next run + // retries. `present` still contains it, so it is not mistaken + // for a deletion. + failed++ } - out = append(out, item) } - // Deletion detection (incremental only) — based on logical_key vanishing, - // so a same-name replacement (media_id churns but logical_key stays) - // never fires a spurious delete. + // Deletion detection (incremental only) — based on logical_key vanishing + // from the listing, so neither a same-name replacement (media_id churns + // but logical_key stays) nor a failed download fires a spurious delete. deleted := 0 if incremental && prev != nil && prev.KBLogical != nil { if prevSet, ok := prev.KBLogical[kbID]; ok { for prevKey := range prevSet { - if _, still := currentLogical[prevKey]; !still { + if _, still := present[prevKey]; !still { out = append(out, types.FetchedItem{ ExternalID: prevKey, IsDeleted: true, @@ -306,8 +317,8 @@ func (c *Connector) walk( } } - logger.Infof(ctx, "[IMA] KB %s: total=%d replaced=%d deleted=%d", - kbID, len(files), replaced, deleted) + logger.Infof(ctx, "[IMA] KB %s: total=%d replaced=%d failed=%d deleted=%d", + kbID, len(files), replaced, failed, deleted) } if !incremental { @@ -396,50 +407,83 @@ func listAllKBFiles( return out, folderPath, nil } +// fetchOutcome distinguishes the three ways fetchOneMedia can end, because the +// caller has to treat them differently when building the sync cursor: only a +// transient failure must stay out of the cursor so it is retried next run. +type fetchOutcome int + +const ( + // fetchOK: the item was resolved and should be ingested. + fetchOK fetchOutcome = iota + // fetchSkipped: IMA cannot give us this item's content and never will + // (unsupported media type, no URL). Deterministic, so it is safe to + // remember in the cursor and stop re-probing it. + fetchSkipped + // fetchFailed: a transient error (API call or download). Must be retried. + fetchFailed +) + // fetchOneMedia calls get_media_info for a single item and, when possible, -// downloads its body. Returns false if the item should be skipped (unsupported -// media type, download error, note-type, etc.). +// downloads its body. // // externalID is the stable logical_key computed by the caller — same value // across syncs even after an IMA same-name replacement mints a new media_id. // The raw media_id is still preserved in metadata for observability. func fetchOneMedia( - ctx context.Context, cli *client, cfg *Config, + ctx context.Context, cli *client, kbID string, externalID string, f walkedFile, folderPath map[string]string, -) (types.FetchedItem, bool) { +) (types.FetchedItem, fetchOutcome) { info, err := cli.GetMediaInfo(ctx, f.MediaID) if err != nil { - logger.Warnf(ctx, "[IMA] get_media_info(%s) failed, skipping: %v", f.MediaID, err) - return types.FetchedItem{}, false + logger.Warnf(ctx, "[IMA] get_media_info(%s) failed, will retry next sync: %v", f.MediaID, err) + return types.FetchedItem{}, fetchFailed } if isSkippableMediaType(info.MediaType) { logger.Infof(ctx, "[IMA] skip media %s (title=%q media_type=%d): unsupported by wiki OpenAPI", f.MediaID, f.Title, info.MediaType) - return types.FetchedItem{}, false + return types.FetchedItem{}, fetchSkipped } if info.URLInfo.URL == "" { logger.Warnf(ctx, "[IMA] media %s (title=%q) has no url_info.url, skipping", f.MediaID, f.Title) - return types.FetchedItem{}, false + return types.FetchedItem{}, fetchSkipped } ext := extensionForMediaType(info.MediaType) + + // A media type with no fixed extension (web page, WeChat article, ...) is + // normally handed to the ingest layer as a bare URL so WeKnora fetches it + // itself. That only works for publicly reachable links: when IMA attaches + // auth headers the URL points at IMA-hosted storage and WeKnora's own + // fetch — which cannot carry those headers — would be rejected. In that + // case download here, where the headers are available, and infer the + // extension from the response instead. if ext == "" { - return types.FetchedItem{ - ExternalID: externalID, - Title: f.Title, - URL: info.URLInfo.URL, - SourceResourceID: kbID, - UpdatedAt: time.Now(), - Metadata: baseMetadata(externalID, f, folderPath, info, kbID), - }, true + if len(info.URLInfo.Headers) == 0 { + return types.FetchedItem{ + ExternalID: externalID, + Title: f.Title, + URL: info.URLInfo.URL, + SourceResourceID: kbID, + UpdatedAt: time.Now(), + Metadata: baseMetadata(externalID, f, folderPath, info, kbID), + }, fetchOK + } + ext = "html" } body, ct, err := cli.DownloadURL(ctx, info.URLInfo) if err != nil { - logger.Warnf(ctx, "[IMA] download media %s (title=%q) failed, skipping: %v", f.MediaID, f.Title, err) - return types.FetchedItem{}, false + logger.Warnf(ctx, "[IMA] download media %s (title=%q) failed, will retry next sync: %v", + f.MediaID, f.Title, err) + return types.FetchedItem{}, fetchFailed + } + + // IMA labels every image media_type=9, so trust the response's own + // Content-Type over the media_type default when it names a real format. + if detected := extensionForContentType(ct); detected != "" { + ext = detected } if ct == "" || strings.HasPrefix(ct, "application/octet-stream") { ct = mimeForExtension(ext) @@ -460,14 +504,17 @@ func fetchOneMedia( UpdatedAt: time.Now(), SourceResourceID: kbID, Metadata: baseMetadata(externalID, f, folderPath, info, kbID), - }, true + }, fetchOK } // baseMetadata builds the metadata map preserved on every ingested item. // externalID (the caller's logical_key) is stored so operators can join the // stable identity back to the raw media_id — useful when debugging why a // same-name replacement showed up as an update instead of an add. -func baseMetadata(externalID string, f walkedFile, folderPath map[string]string, info *getMediaInfoResp, kbID string) map[string]string { +func baseMetadata( + externalID string, f walkedFile, folderPath map[string]string, + info *getMediaInfoResp, kbID string, +) map[string]string { m := map[string]string{ "channel": types.ChannelIMA, "media_id": f.MediaID, diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.go new file mode 100644 index 00000000..e724a77d --- /dev/null +++ b/internal/datasource/connector/ima/connector_test.go @@ -0,0 +1,404 @@ +package ima + +import ( + "context" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" +) + +func TestConnector_Type(t *testing.T) { + if got := NewConnector().Type(); got != types.ConnectorTypeIMA { + t.Errorf("Type() = %q, want %q", got, types.ConnectorTypeIMA) + } +} + +// TestFetchIncremental_TransientFailureIsRetried is the regression test for the +// cursor bug: the cursor used to be built from the raw listing before any +// content was fetched, so an item whose download failed was recorded as +// "seen with this media_id" and every later incremental sync skipped it as +// unchanged. The document was then silently missing from the KB forever. +func TestFetchIncremental_TransientFailureIsRetried(t *testing.T) { + f := newFakeIMA(t) + good := fakeFile{MediaID: "m-good", Title: "Good", MediaType: mediaTypeMarkdown, Body: "# ok"} + flaky := fakeFile{ + MediaID: "m-flaky", Title: "Flaky", MediaType: mediaTypeMarkdown, + Body: "# later", DownloadFails: true, + } + f.setKB("kb1", []fakeFile{good, flaky}) + + c := NewConnector() + cfg := f.config("kb1") + goodKey := logicalKey("kb1", "", "Good") + flakyKey := logicalKey("kb1", "", "Flaky") + + items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + if _, ok := findItem(items, flakyKey); ok { + t.Fatalf("failed download must not be emitted, got %s", describeItems(items)) + } + mustFindItem(t, items, goodKey) + + state := decodeCursor(t, cursor) + if _, recorded := state.KBLogical["kb1"][flakyKey]; recorded { + t.Fatal("a failed item must not be recorded in the cursor, otherwise it is never retried") + } + if state.KBLogical["kb1"][goodKey] != "m-good" { + t.Fatalf("successful item missing from cursor: %#v", state.KBLogical["kb1"]) + } + + // Second pass: the download now succeeds and the item must be re-attempted. + flaky.DownloadFails = false + f.setKB("kb1", []fakeFile{good, flaky}) + + items2, cursor2, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + retried := mustFindItem(t, items2, flakyKey) + if string(retried.Content) != "# later" { + t.Errorf("retried content = %q, want %q", retried.Content, "# later") + } + if _, ok := findItem(items2, goodKey); ok { + t.Errorf("unchanged item must be skipped on the second pass, got %s", describeItems(items2)) + } + if decodeCursor(t, cursor2).KBLogical["kb1"][flakyKey] != "m-flaky" { + t.Error("the retried item should now be recorded in the cursor") + } +} + +// TestFetchIncremental_FailureIsNotADeletion guards the other half of the fix: +// keeping a failed item out of the cursor must not make the next sync mistake +// it for a document that vanished from IMA. +func TestFetchIncremental_FailureIsNotADeletion(t *testing.T) { + f := newFakeIMA(t) + good := fakeFile{MediaID: "m-good", Title: "Good", MediaType: mediaTypeMarkdown, Body: "# ok"} + flaky := fakeFile{MediaID: "m-flaky", Title: "Flaky", MediaType: mediaTypeMarkdown, DownloadFails: true} + f.setKB("kb1", []fakeFile{good, flaky}) + + c := NewConnector() + cfg := f.config("kb1") + flakyKey := logicalKey("kb1", "", "Flaky") + + _, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + + // Still failing, still listed by IMA: no tombstone may be emitted. + items, _, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + for _, it := range items { + if it.IsDeleted && it.ExternalID == flakyKey { + t.Fatalf("a still-listed item that merely failed to download must not be tombstoned: %s", + describeItems(items)) + } + } +} + +func TestFetchIncremental_RemovedItemIsTombstoned(t *testing.T) { + f := newFakeIMA(t) + keep := fakeFile{MediaID: "m-keep", Title: "Keep", MediaType: mediaTypeMarkdown, Body: "# keep"} + drop := fakeFile{MediaID: "m-drop", Title: "Drop", MediaType: mediaTypeMarkdown, Body: "# drop"} + f.setKB("kb1", []fakeFile{keep, drop}) + + c := NewConnector() + cfg := f.config("kb1") + dropKey := logicalKey("kb1", "", "Drop") + + _, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + + f.setKB("kb1", []fakeFile{keep}) + items, _, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + + tombstone, ok := findItem(items, dropKey) + if !ok || !tombstone.IsDeleted { + t.Fatalf("removed item must be tombstoned, got %s", describeItems(items)) + } + if tombstone.SourceResourceID != "kb1" { + t.Errorf("tombstone SourceResourceID = %q, want kb1", tombstone.SourceResourceID) + } +} + +// TestFetchIncremental_SameNameReplacementKeepsExternalID covers IMA minting a +// new media_id when a file is replaced in place: the item must come back as an +// update under the same external id rather than a delete plus an insert. +func TestFetchIncremental_SameNameReplacementKeepsExternalID(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-v1", Title: "Doc", MediaType: mediaTypeMarkdown, Body: "v1"}}) + + c := NewConnector() + cfg := f.config("kb1") + key := logicalKey("kb1", "", "Doc") + + _, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + + f.setKB("kb1", []fakeFile{{MediaID: "m-v2", Title: "Doc", MediaType: mediaTypeMarkdown, Body: "v2"}}) + items, cursor2, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + + updated := mustFindItem(t, items, key) + if updated.IsDeleted { + t.Fatal("a same-name replacement must be an update, not a delete") + } + if string(updated.Content) != "v2" { + t.Errorf("content = %q, want v2", updated.Content) + } + if updated.Metadata["media_id"] != "m-v2" { + t.Errorf("metadata media_id = %q, want m-v2", updated.Metadata["media_id"]) + } + for _, it := range items { + if it.IsDeleted { + t.Errorf("no tombstone expected, got %s", describeItems(items)) + } + } + if decodeCursor(t, cursor2).KBLogical["kb1"][key] != "m-v2" { + t.Error("cursor should track the new media_id") + } +} + +// TestFetchIncremental_UnsupportedTypeIsProbedOnce checks that a deterministic +// skip is remembered, so an IMA knowledge base full of notes does not re-pay +// for get_media_info on every scheduled sync. +func TestFetchIncremental_UnsupportedTypeIsProbedOnce(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-note", Title: "Note", MediaType: mediaTypeNote}}) + + c := NewConnector() + cfg := f.config("kb1") + + items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + if len(items) != 0 { + t.Fatalf("note must be skipped, got %s", describeItems(items)) + } + if f.callCount("get_media_info:m-note") != 1 { + t.Fatalf("get_media_info calls = %d, want 1", f.callCount("get_media_info:m-note")) + } + + if _, _, err := c.FetchIncremental(context.Background(), cfg, cursor); err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + if got := f.callCount("get_media_info:m-note"); got != 1 { + t.Errorf("get_media_info calls after second sync = %d, want 1 (skip should be cached)", got) + } +} + +// TestFetchAll_AuthenticatedURLIsDownloaded covers media types with no fixed +// extension. When IMA attaches auth headers the URL is IMA-hosted and WeKnora's +// own fetch could not authenticate, so the connector must download it here and +// forward the headers. +func TestFetchAll_AuthenticatedURLIsDownloaded(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{ + MediaID: "m-web", + Title: "Article", + MediaType: mediaTypeWeb, + Body: "body", + ContentType: "text/html", + URLHeaders: map[string]string{"X-Ima-Token": "secret"}, + }}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + item := mustFindItem(t, items, logicalKey("kb1", "", "Article")) + if string(item.Content) != "body" { + t.Errorf("content = %q, want the downloaded body", item.Content) + } + if !strings.HasSuffix(item.FileName, ".html") { + t.Errorf("FileName = %q, want an .html suffix", item.FileName) + } + if got := f.downloadHeaders["m-web"].Get("X-Ima-Token"); got != "secret" { + t.Errorf("auth header forwarded = %q, want %q", got, "secret") + } +} + +// TestFetchAll_PublicURLStaysURLOnly is the counterpart: without auth headers +// the link is publicly reachable, so it is handed to the ingest layer as a URL +// and WeKnora renders the live page instead of a snapshot. +func TestFetchAll_PublicURLStaysURLOnly(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-web", Title: "Article", MediaType: mediaTypeWeb, Body: "ignored"}}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + item := mustFindItem(t, items, logicalKey("kb1", "", "Article")) + if len(item.Content) != 0 { + t.Errorf("public URL must not be downloaded, got %d bytes", len(item.Content)) + } + if item.URL == "" { + t.Error("URL-only item must carry a URL") + } + if f.callCount("download:m-web") != 0 { + t.Error("no download should have been attempted") + } +} + +// TestFetchAll_ImageExtensionFollowsContentType guards against naming a JPEG +// ".png": IMA reports every image as media_type=9, so only the download's +// Content-Type reveals the real format, and the extension drives the ingest +// pipeline's file-type checks. +func TestFetchAll_ImageExtensionFollowsContentType(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{ + MediaID: "m-img", + Title: "Photo", + MediaType: mediaTypeImage, + Body: "\xff\xd8\xff\xe0jpeg", + ContentType: "image/jpeg", + }}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + item := mustFindItem(t, items, logicalKey("kb1", "", "Photo")) + if !strings.HasSuffix(item.FileName, ".jpg") { + t.Errorf("FileName = %q, want a .jpg suffix for an image/jpeg body", item.FileName) + } +} + +func TestFetchAll_SkipsItemWithoutURL(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-empty", Title: "Empty", MediaType: mediaTypePDF, NoURL: true}}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + if len(items) != 0 { + t.Fatalf("item without a URL must be skipped, got %s", describeItems(items)) + } +} + +// TestFetchAll_ResolvesFolderPath checks the mixed file/folder listing walk. +func TestFetchAll_ResolvesFolderPath(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", + []fakeFile{ + {MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"}, + {MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"}, + }, + fakeFolder{FolderID: "f1", Name: "Outer"}, + fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"}, + ) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + + deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep")) + if deep.Metadata["folder_path"] != "Outer/Inner" { + t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"]) + } + root := mustFindItem(t, items, logicalKey("kb1", "", "Root")) + if root.Metadata["folder_path"] != "" { + t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"]) + } +} + +func TestListResources_ReturnsAddableBases(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb-b", nil) + f.setKB("kb-a", nil) + + resources, err := NewConnector().ListResources(context.Background(), f.config(), "") + if err != nil { + t.Fatalf("ListResources: %v", err) + } + if len(resources) != 2 { + t.Fatalf("got %d resources, want 2", len(resources)) + } + if resources[0].ExternalID != "kb-a" || resources[1].ExternalID != "kb-b" { + t.Errorf("resources are not sorted by id: %+v", resources) + } + if resources[0].Description != "desc kb-a" { + t.Errorf("description not enriched from get_knowledge_base: %q", resources[0].Description) + } +} + +// TestListResources_FallsBackToSearch covers tenants whose credential exposes +// read-only scopes, where get_addable_knowledge_base_list comes back empty. +func TestListResources_FallsBackToSearch(t *testing.T) { + f := newFakeIMA(t) + f.searchBases = []searchedKnowledgeBaseInfo{{ID: "kb-x", Name: "X", CoverURL: "https://cover"}} + + resources, err := NewConnector().ListResources(context.Background(), f.config(), "") + if err != nil { + t.Fatalf("ListResources: %v", err) + } + if len(resources) != 1 || resources[0].ExternalID != "kb-x" { + t.Fatalf("expected the search fallback to surface kb-x, got %+v", resources) + } + if resources[0].Metadata["cover_url"] != "https://cover" { + t.Errorf("cover_url = %v, want https://cover", resources[0].Metadata["cover_url"]) + } +} + +func TestListResources_ChildrenAreEmpty(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", nil) + + resources, err := NewConnector().ListResources(context.Background(), f.config(), "kb1") + if err != nil { + t.Fatalf("ListResources: %v", err) + } + if len(resources) != 0 { + t.Errorf("knowledge bases have no children, got %+v", resources) + } +} + +func TestValidate_InvalidCredentials(t *testing.T) { + f := newFakeIMA(t) + cfg := f.config() + cfg.Credentials["client_id"] = "" + + err := NewConnector().Validate(context.Background(), cfg) + if err == nil { + t.Fatal("expected an error for a missing client_id") + } + if !strings.Contains(err.Error(), "client_id") { + t.Errorf("error = %v, want it to name the missing field", err) + } +} + +func TestFetchIncremental_RequiresResourceIDs(t *testing.T) { + f := newFakeIMA(t) + if _, _, err := NewConnector().FetchIncremental(context.Background(), f.config(), nil); err == nil { + t.Fatal("expected an error when no knowledge base is selected") + } +} + +func TestConnectorIsRegisteredInMetadata(t *testing.T) { + meta, ok := datasource.ConnectorMetadataRegistry[types.ConnectorTypeIMA] + if !ok { + t.Fatal("IMA connector missing from the metadata registry") + } + if meta.Type != types.ConnectorTypeIMA { + t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA) + } +} diff --git a/internal/datasource/connector/ima/fake_ima_test.go b/internal/datasource/connector/ima/fake_ima_test.go new file mode 100644 index 00000000..5183c0c3 --- /dev/null +++ b/internal/datasource/connector/ima/fake_ima_test.go @@ -0,0 +1,313 @@ +package ima + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1) +// are reachable. Production keeps the default strict SSRF policy. +func TestMain(m *testing.M) { + _ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + secutils.ResetSSRFWhitelistForTest() + os.Exit(m.Run()) +} + +// fakeFile is one entry in a fake knowledge base listing. +type fakeFile struct { + MediaID string + Title string + ParentFolderID string + + // MediaType is what get_media_info reports for this file. + MediaType int32 + // Body is what the download URL serves. + Body string + // ContentType is the download response's Content-Type ("" omits the header). + ContentType string + // URLHeaders are the auth headers IMA attaches to url_info. + URLHeaders map[string]string + // NoURL makes get_media_info return an empty url_info.url. + NoURL bool + + // InfoFails makes get_media_info return a 500 for this media_id. + InfoFails bool + // DownloadFails makes the download URL return a 500. + DownloadFails bool +} + +// fakeFolder is a folder entry in a fake knowledge base listing. +type fakeFolder struct { + FolderID string + Name string + ParentFolderID string +} + +// fakeIMA is an in-process stand-in for the IMA OpenAPI. Only the endpoints +// the connector calls are implemented. +type fakeIMA struct { + server *httptest.Server + + mu sync.Mutex + // files and folders are keyed by knowledge base id, then by parent folder + // id ("" for the KB root). + files map[string]map[string][]fakeFile + folders map[string]map[string][]fakeFolder + // bases is what get_addable_knowledge_base_list returns. + bases []addableKnowledgeBaseInfo + // searchBases is what search_knowledge_base returns. + searchBases []searchedKnowledgeBaseInfo + + // calls counts requests per action, plus "download:". + calls map[string]int + // downloadHeaders records the headers seen by the last download of a media. + downloadHeaders map[string]http.Header +} + +func newFakeIMA(t *testing.T) *fakeIMA { + t.Helper() + f := &fakeIMA{ + files: map[string]map[string][]fakeFile{}, + folders: map[string]map[string][]fakeFolder{}, + calls: map[string]int{}, + downloadHeaders: map[string]http.Header{}, + } + mux := http.NewServeMux() + mux.HandleFunc(apiBasePath+"/", f.handleAPI) + mux.HandleFunc("/dl/", f.handleDownload) + f.server = httptest.NewServer(mux) + t.Cleanup(f.server.Close) + return f +} + +// setKB replaces the listing of a knowledge base and registers it as addable. +func (f *fakeIMA) setKB(kbID string, files []fakeFile, folders ...fakeFolder) { + f.mu.Lock() + defer f.mu.Unlock() + + byParent := map[string][]fakeFile{} + for _, file := range files { + byParent[file.ParentFolderID] = append(byParent[file.ParentFolderID], file) + } + f.files[kbID] = byParent + + foldersByParent := map[string][]fakeFolder{} + for _, folder := range folders { + foldersByParent[folder.ParentFolderID] = append(foldersByParent[folder.ParentFolderID], folder) + } + f.folders[kbID] = foldersByParent + + for _, b := range f.bases { + if b.ID == kbID { + return + } + } + f.bases = append(f.bases, addableKnowledgeBaseInfo{ID: kbID, Name: "KB " + kbID}) +} + +func (f *fakeIMA) callCount(key string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls[key] +} + +func (f *fakeIMA) record(key string) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls[key]++ +} + +// findFile locates a file by media_id across every knowledge base. +func (f *fakeIMA) findFile(mediaID string) (fakeFile, bool) { + f.mu.Lock() + defer f.mu.Unlock() + for _, byParent := range f.files { + for _, files := range byParent { + for _, file := range files { + if file.MediaID == mediaID { + return file, true + } + } + } + } + return fakeFile{}, false +} + +func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) { + action := strings.TrimPrefix(r.URL.Path, apiBasePath+"/") + f.record(action) + + var req map[string]interface{} + _ = json.NewDecoder(r.Body).Decode(&req) + str := func(k string) string { + if v, ok := req[k].(string); ok { + return v + } + return "" + } + + switch action { + case "get_addable_knowledge_base_list": + f.mu.Lock() + resp := getAddableKnowledgeBaseListResp{AddableKnowledgeBaseList: f.bases, IsEnd: true} + f.mu.Unlock() + writeEnvelope(w, 0, "", resp) + + case "search_knowledge_base": + f.mu.Lock() + resp := searchKnowledgeBaseResp{InfoList: f.searchBases, IsEnd: true} + f.mu.Unlock() + writeEnvelope(w, 0, "", resp) + + case "get_knowledge_base": + infos := map[string]knowledgeBaseInfo{} + if ids, ok := req["ids"].([]interface{}); ok { + for _, raw := range ids { + id, _ := raw.(string) + infos[id] = knowledgeBaseInfo{ID: id, Name: "KB " + id, Description: "desc " + id} + } + } + writeEnvelope(w, 0, "", getKnowledgeBaseResp{Infos: infos}) + + case "get_knowledge_list": + kbID, folderID := str("knowledge_base_id"), str("folder_id") + f.mu.Lock() + files := f.files[kbID][folderID] + folders := f.folders[kbID][folderID] + f.mu.Unlock() + + var list []json.RawMessage + for _, folder := range folders { + b, _ := json.Marshal(folderInfo{ + FolderID: folder.FolderID, + Name: folder.Name, + ParentFolderID: folder.ParentFolderID, + }) + list = append(list, b) + } + for _, file := range files { + // IMA omits media_type in list responses. + b, _ := json.Marshal(map[string]interface{}{ + "media_id": file.MediaID, + "title": file.Title, + "parent_folder_id": file.ParentFolderID, + }) + list = append(list, b) + } + writeEnvelope(w, 0, "", getKnowledgeListResp{KnowledgeList: list, IsEnd: true}) + + case "get_media_info": + mediaID := str("media_id") + f.record("get_media_info:" + mediaID) + file, ok := f.findFile(mediaID) + if !ok { + writeEnvelope(w, 110001, "unknown media", nil) + return + } + if file.InfoFails { + w.WriteHeader(http.StatusInternalServerError) + return + } + resp := getMediaInfoResp{MediaType: file.MediaType} + if !file.NoURL { + resp.URLInfo = urlInfo{ + URL: f.server.URL + "/dl/" + mediaID, + Headers: file.URLHeaders, + } + } + writeEnvelope(w, 0, "", resp) + + default: + writeEnvelope(w, 110001, "unsupported action "+action, nil) + } +} + +func (f *fakeIMA) handleDownload(w http.ResponseWriter, r *http.Request) { + mediaID := strings.TrimPrefix(r.URL.Path, "/dl/") + f.record("download:" + mediaID) + + f.mu.Lock() + f.downloadHeaders[mediaID] = r.Header.Clone() + f.mu.Unlock() + + file, ok := f.findFile(mediaID) + if !ok || file.DownloadFails { + w.WriteHeader(http.StatusInternalServerError) + return + } + if file.ContentType != "" { + w.Header().Set("Content-Type", file.ContentType) + } + _, _ = w.Write([]byte(file.Body)) +} + +func writeEnvelope(w http.ResponseWriter, code int, msg string, data interface{}) { + raw, _ := json.Marshal(data) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(apiEnvelope{Code: code, Msg: msg, Data: raw}) +} + +func (f *fakeIMA) config(resourceIDs ...string) *types.DataSourceConfig { + return &types.DataSourceConfig{ + Type: types.ConnectorTypeIMA, + Credentials: map[string]interface{}{ + "client_id": "cid", + "api_key": "key", + "base_url": f.server.URL, + }, + ResourceIDs: resourceIDs, + } +} + +// decodeCursor converts the connector cursor back into its typed form. +func decodeCursor(t *testing.T, cur *types.SyncCursor) *imaCursor { + t.Helper() + if cur == nil { + t.Fatal("cursor is nil") + } + b, err := json.Marshal(cur.ConnectorCursor) + if err != nil { + t.Fatalf("marshal cursor: %v", err) + } + var out imaCursor + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("unmarshal cursor: %v", err) + } + return &out +} + +// findItem returns the fetched item with the given external id. +func findItem(items []types.FetchedItem, externalID string) (types.FetchedItem, bool) { + for _, it := range items { + if it.ExternalID == externalID { + return it, true + } + } + return types.FetchedItem{}, false +} + +func mustFindItem(t *testing.T, items []types.FetchedItem, externalID string) types.FetchedItem { + t.Helper() + it, ok := findItem(items, externalID) + if !ok { + t.Fatalf("item %s not found in %s", externalID, describeItems(items)) + } + return it +} + +func describeItems(items []types.FetchedItem) string { + var parts []string + for _, it := range items { + parts = append(parts, fmt.Sprintf("{id=%s title=%q deleted=%t}", it.ExternalID, it.Title, it.IsDeleted)) + } + return "[" + strings.Join(parts, " ") + "]" +} diff --git a/internal/datasource/connector/ima/types.go b/internal/datasource/connector/ima/types.go index 5c8feee1..7c3f3298 100644 --- a/internal/datasource/connector/ima/types.go +++ b/internal/datasource/connector/ima/types.go @@ -202,21 +202,22 @@ type getMediaInfoResp struct { // imaCursor tracks per-knowledge-base state seen during the last sync. // -// Two parallel maps are kept, both keyed by kb_id: +// KBLogical: { logical_key: media_id } — the authoritative source for both +// delete detection AND "replacement" detection. IMA reassigns media_id +// whenever a same-named file is replaced in place, so a logical key (see +// logicalKey) is what we treat as a document's stable identity. +// A logical_key present last sync but absent this sync ⇒ delete. +// A logical_key whose media_id changed ⇒ replacement (re-fetch content; the +// emitted FetchedItem carries the same ExternalID so the ingest layer's +// "existing external_id → delete-and-recreate" path (see +// datasource_service.go applyFetchedItem) treats it as an update. // -// - KBLogical: { logical_key: media_id } — the authoritative source for -// both delete detection AND "replacement" detection. IMA reassigns -// media_id whenever a same-named file is replaced in place, so a logical -// key (see logicalKey) is what we treat as a document's stable identity. -// A logical_key present last sync but absent this sync ⇒ delete. -// A logical_key whose media_id changed ⇒ replacement (re-fetch content; -// the emitted FetchedItem carries the same ExternalID so the ingest -// layer's "existing external_id → delete-and-recreate" path (see -// datasource_service.go applyFetchedItem) treats it as an update. +// Only items that were successfully synced (or deterministically skipped) +// this cycle are recorded, so a transient download failure leaves the key +// absent and the next run retries it instead of treating it as unchanged. // -// - KBMedia (legacy): { media_id: title } — retained so cursors persisted -// by older versions still deserialize; not consulted for new detection. -// New cursors write this too, purely for observability in logs. +// KBMedia is a legacy field retained only so cursors persisted by earlier +// builds still deserialize; it is never written or consulted. type imaCursor struct { LastSyncTime time.Time `json:"last_sync_time"` KBLogical map[string]map[string]string `json:"kb_logical,omitempty"` @@ -226,8 +227,8 @@ type imaCursor struct { // logicalKey derives a stable identity for an IMA item that survives // same-name replacement (which mints a new media_id server-side). // -// The key is a short SHA-256 hex over (kb_id, parent_folder_id, title, -// media_type). Rationale: +// The key is a short SHA-256 hex over (kb_id, parent_folder_id, title). +// Rationale: // // - kb_id keeps identities isolated across knowledge bases (same file // copied into two KBs is two distinct docs from WeKnora's perspective). @@ -235,17 +236,20 @@ type imaCursor struct { // display name — the tuple IMA's own check_repeated_names uses to gate // uploads (see api.md), so IMA's own uniqueness constraint enforces // this key's uniqueness inside a KB. -// - media_type distinguishes "notes.md" (Markdown) from "notes.md" -// (Word) in the same folder — pathologically rare but cheap to guard. +// +// media_type is deliberately NOT part of the key: get_knowledge_list omits it +// (it only arrives later from get_media_info), so folding it in would mean +// every external_id shifts the day IMA starts populating the field in list +// responses — which reads as "every document deleted and re-added". // // Emitted as "ima_" + first 32 hex chars (128 bits) — small enough to fit // well within Postgres varchar(64) external_id columns, still >> collision // resistant for any realistic KB size. -func logicalKey(kbID, parentFolderID, title string, mediaType int32) string { +func logicalKey(kbID, parentFolderID, title string) string { h := sha256.New() // Delimiter can't appear in any component because it's neither a valid // IMA id character nor allowed in titles once we've folded to bytes. - fmt.Fprintf(h, "%s\x1f%s\x1f%s\x1f%d", kbID, parentFolderID, title, mediaType) + _, _ = fmt.Fprintf(h, "%s\x1f%s\x1f%s", kbID, parentFolderID, title) return "ima_" + hex.EncodeToString(h.Sum(nil))[:32] } @@ -301,6 +305,35 @@ func extensionForMediaType(t int32) string { } } +// extensionForContentType maps a response Content-Type back to a file +// extension, ignoring any "; charset=..." suffix. Returns "" for types we +// have no better guess for than the media_type-derived default. +// +// IMA reports every image as media_type=9 regardless of the real encoding, so +// the download's Content-Type is the only signal that distinguishes a JPEG +// from a PNG. Naming a JPEG ".png" breaks the extension-based file-type checks +// downstream in the ingest pipeline. +func extensionForContentType(contentType string) string { + mediaType := strings.ToLower(strings.TrimSpace(contentType)) + if i := strings.Index(mediaType, ";"); i >= 0 { + mediaType = strings.TrimSpace(mediaType[:i]) + } + switch mediaType { + case "image/png": + return "png" + case "image/jpeg", "image/jpg": + return "jpg" + case "image/gif": + return "gif" + case "image/webp": + return "webp" + case "image/bmp": + return "bmp" + default: + return "" + } +} + // mimeForExtension maps a file extension to a canonical MIME type. // Used when the downloaded URL response omits (or misreports) Content-Type. func mimeForExtension(ext string) string { @@ -331,6 +364,10 @@ func mimeForExtension(ext string) string { return "image/jpeg" case "webp": return "image/webp" + case "gif": + return "image/gif" + case "bmp": + return "image/bmp" case "epub": return "application/epub+zip" case "xmind": diff --git a/internal/datasource/connector/ima/types_test.go b/internal/datasource/connector/ima/types_test.go new file mode 100644 index 00000000..d325b275 --- /dev/null +++ b/internal/datasource/connector/ima/types_test.go @@ -0,0 +1,163 @@ +package ima + +import ( + "errors" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" +) + +func TestLogicalKey_IsStableAndScoped(t *testing.T) { + base := logicalKey("kb1", "f1", "Doc") + + if base != logicalKey("kb1", "f1", "Doc") { + t.Error("logicalKey must be deterministic") + } + if !strings.HasPrefix(base, "ima_") || len(base) != 36 { + t.Errorf("key = %q, want an ima_ prefix and 36 chars so it fits varchar(64)", base) + } + + for _, other := range []string{ + logicalKey("kb2", "f1", "Doc"), + logicalKey("kb1", "f2", "Doc"), + logicalKey("kb1", "f1", "Other"), + } { + if other == base { + t.Errorf("keys must differ across kb/folder/title, both were %q", base) + } + } +} + +// TestLogicalKey_DelimiterIsUnambiguous guards against a key collision from +// naively concatenating the components. +func TestLogicalKey_DelimiterIsUnambiguous(t *testing.T) { + if logicalKey("kb", "a", "bc") == logicalKey("kb", "ab", "c") { + t.Error("component boundaries must not be ambiguous") + } +} + +func TestParseIMAConfig(t *testing.T) { + tests := []struct { + name string + credentials map[string]interface{} + wantErr error + wantErrText string + }{ + { + name: "valid", + credentials: map[string]interface{}{"client_id": "cid", "api_key": "key"}, + }, + { + name: "missing client_id", + credentials: map[string]interface{}{"api_key": "key"}, + wantErr: datasource.ErrInvalidCredentials, + }, + { + name: "blank api_key", + credentials: map[string]interface{}{"client_id": "cid", "api_key": " "}, + wantErr: datasource.ErrInvalidCredentials, + }, + { + name: "loopback base_url rejected by SSRF policy", + credentials: map[string]interface{}{"client_id": "cid", "api_key": "key", "base_url": "http://169.254.169.254"}, + wantErrText: "SSRF", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseIMAConfig(&types.DataSourceConfig{Credentials: tt.credentials}) + switch { + case tt.wantErr != nil: + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + case tt.wantErrText != "": + if err == nil || !strings.Contains(err.Error(), tt.wantErrText) { + t.Fatalf("err = %v, want it to mention %q", err, tt.wantErrText) + } + default: + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + }) + } +} + +func TestParseIMAConfig_NilConfig(t *testing.T) { + if _, err := parseIMAConfig(nil); !errors.Is(err, datasource.ErrInvalidConfig) { + t.Fatalf("err = %v, want ErrInvalidConfig", err) + } +} + +func TestConfig_GetBaseURL(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"", DefaultBaseURL}, + {" ", DefaultBaseURL}, + {"https://ima.example.com/", "https://ima.example.com"}, + {"ima.example.com", "https://ima.example.com"}, + {"http://ima.example.com//", "http://ima.example.com"}, + } + for _, tt := range tests { + if got := (&Config{BaseURL: tt.in}).GetBaseURL(); got != tt.want { + t.Errorf("GetBaseURL(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestExtensionForContentType(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"image/jpeg", "jpg"}, + {"image/JPEG", "jpg"}, + {"image/png; charset=binary", "png"}, + {"image/webp", "webp"}, + {"application/pdf", ""}, + {"", ""}, + } + for _, tt := range tests { + if got := extensionForContentType(tt.in); got != tt.want { + t.Errorf("extensionForContentType(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestSanitizeFileName(t *testing.T) { + if got := sanitizeFileName(""); got != "untitled" { + t.Errorf("empty name = %q, want untitled", got) + } + if got := sanitizeFileName(`a/b\c:d*e?f"gi|j`); strings.ContainsAny(got, `/\:*?"<>|`) { + t.Errorf("filesystem-hostile characters survived: %q", got) + } + + // A long CJK title must be truncated on a rune boundary, not mid-sequence. + long := strings.Repeat("知识", 200) + got := sanitizeFileName(long) + if len(got) > 200 { + t.Errorf("len = %d bytes, want <= 200", len(got)) + } + if !strings.HasPrefix(long, got) { + t.Errorf("truncation corrupted the prefix: %q", got) + } +} + +func TestIsSkippableMediaType(t *testing.T) { + for _, mt := range []int32{mediaTypeNote, mediaTypeAISession, mediaTypeVideo} { + if !isSkippableMediaType(mt) { + t.Errorf("media type %d should be skippable", mt) + } + } + for _, mt := range []int32{mediaTypePDF, mediaTypeMarkdown, mediaTypeWeb} { + if isSkippableMediaType(mt) { + t.Errorf("media type %d should not be skippable", mt) + } + } +}