From 3d645d2578c65e00a7f61aaa9115046a866b8a10 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 12 Aug 2026 14:38:16 +0800 Subject: [PATCH] fix(datasource/ima): retry transient sync failures instead of losing documents The sync cursor was built from the raw IMA listing before any content was fetched, so an item whose get_media_info or download failed was still recorded as "seen with this media_id". Every later incremental sync then skipped it as unchanged and the document was silently missing from the knowledge base for good. Track the listed keys and the successfully synced keys separately: the cursor now only remembers what was actually resolved, while delete detection keeps using the full listing so a failed download is never mistaken for a document that vanished. Deterministic skips (unsupported media types, items with no URL) are still recorded so they are not re-probed every cycle. Alongside that: - Drop media_type from logicalKey. get_knowledge_list omits the field, so folding it in meant every external_id would shift the day IMA starts populating it, which reads downstream as "every document deleted". - Download URL-backed media that carry auth headers rather than handing the bare URL to the ingest layer, which cannot replay those headers. - Derive image extensions from the response Content-Type; IMA reports every image as media_type=9, so a JPEG was being written as .png. - Stop logging API response bodies, which carry pre-signed storage URLs and per-request auth headers. - Validate media URLs against the SSRF policy before the first connection. - Stop writing the redundant kb_media map into the persisted cursor. - Say in the connector description that notes and AI sessions are skipped. --- frontend/src/i18n/locales/en-US.ts | 2 +- frontend/src/i18n/locales/ko-KR.ts | 2 +- frontend/src/i18n/locales/ru-RU.ts | 2 +- frontend/src/i18n/locales/zh-CN.ts | 2 +- internal/datasource/connector/ima/client.go | 29 +- .../datasource/connector/ima/connector.go | 145 ++++--- .../connector/ima/connector_test.go | 404 ++++++++++++++++++ .../datasource/connector/ima/fake_ima_test.go | 313 ++++++++++++++ internal/datasource/connector/ima/types.go | 75 +++- .../datasource/connector/ima/types_test.go | 163 +++++++ 10 files changed, 1058 insertions(+), 79 deletions(-) create mode 100644 internal/datasource/connector/ima/connector_test.go create mode 100644 internal/datasource/connector/ima/fake_ima_test.go create mode 100644 internal/datasource/connector/ima/types_test.go diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 99bdea59..205b3490 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -5486,7 +5486,7 @@ export default { lark_drive: 'Sync documents, spreadsheets and files from a Lark Drive folder (Feishu international)', notion: 'Sync pages and databases from Notion', yuque: 'Sync documents from Yuque knowledge bases', - ima: 'Sync documents, notes and files from Tencent IMA knowledge bases', + ima: 'Sync documents and files from Tencent IMA knowledge bases (notes and AI sessions are not supported yet)', rss: 'Sync articles from RSS / Atom feeds' }, drive: { diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 796cbf9f..32116e89 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -704,7 +704,7 @@ export default { lark_drive: "Lark 드라이브 폴더에서 문서, 스프레드시트, 파일 동기화", notion: 'Notion에서 페이지 및 데이터베이스 동기화', yuque: '위큐 지식베이스에서 문서 동기화', - ima: 'Tencent IMA 지식베이스에서 문서, 노트 및 파일 동기화', + ima: 'Tencent IMA 지식베이스에서 문서 및 파일 동기화 (노트와 AI 세션은 아직 지원되지 않음)', rss: 'RSS / Atom 피드에서 글 동기화' }, connector: { diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index a19624b8..7ac69061 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -704,7 +704,7 @@ export default { lark_drive: 'Синхронизация документов, таблиц и файлов из папки Lark Drive', notion: 'Синхронизация страниц и баз данных из Notion', yuque: 'Синхронизация документов из баз знаний Yuque', - ima: 'Синхронизация документов, заметок и файлов из баз знаний Tencent IMA', + ima: 'Синхронизация документов и файлов из баз знаний Tencent IMA (заметки и ИИ-сессии пока не поддерживаются)', rss: 'Синхронизация статей из лент RSS / Atom' }, connector: { diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index a6bc1778..363c0d5f 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -704,7 +704,7 @@ export default { lark_drive: "同步 Lark 云盘文件夹中的文档、表格、文件(飞书国际版)", notion: '同步 Notion 中的页面和数据库', yuque: '同步语雀知识库中的文档', - ima: '同步腾讯 IMA 知识库中的文档、笔记与文件', + ima: '同步腾讯 IMA 知识库中的文档与文件(暂不支持笔记与 AI 会话)', rss: '同步 RSS / Atom 订阅源中的文章' }, connector: { diff --git a/internal/datasource/connector/ima/client.go b/internal/datasource/connector/ima/client.go index 95a8395e..aef65cda 100644 --- a/internal/datasource/connector/ima/client.go +++ b/internal/datasource/connector/ima/client.go @@ -1,3 +1,6 @@ +// Package ima implements the WeKnora data source connector for Tencent IMA +// (ima.qq.com), syncing documents and files out of IMA knowledge bases via the +// /openapi/wiki/v1 OpenAPI. package ima import ( @@ -12,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/utils" ) const ( @@ -112,7 +116,7 @@ func (c *client) callAPI(ctx context.Context, action string, req interface{}, re } respBody, readErr := io.ReadAll(resp.Body) - resp.Body.Close() + _ = resp.Body.Close() if readErr != nil { lastErr = fmt.Errorf("read response: %w", readErr) if attempt < maxRetries { @@ -124,9 +128,11 @@ func (c *client) callAPI(ctx context.Context, action string, req interface{}, re return lastErr } + // The body is only ever logged on an error path: successful responses + // (get_media_info in particular) carry pre-signed storage URLs and + // per-request auth headers that must not land in the server log. bodyPreview := truncate(string(respBody), 500) - logger.Infof(ctx, "[IMA] POST %s → status=%d bodyLen=%d body=%s", - action, resp.StatusCode, len(respBody), bodyPreview) + logger.Infof(ctx, "[IMA] POST %s → status=%d bodyLen=%d", action, resp.StatusCode, len(respBody)) // HTTP-level auth failures: surface as ErrInvalidCredentials. if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { @@ -167,9 +173,11 @@ func (c *client) callAPI(ctx context.Context, action string, req interface{}, re } // Business-level errors. - // 110030 (无权限) / 110001 (参数非法) with common auth hint patterns are - // mapped to ErrInvalidCredentials so the service marks the source in - // `error` state and stops scheduling until re-authenticated. + // 110030 (无权限) is mapped to ErrInvalidCredentials so the service marks + // the source in `error` state and stops scheduling until re-authenticated. + // 110021 (限频) is retried. Everything else — including 110001 (参数非法), + // which is a bug on our side rather than a credential problem — surfaces + // verbatim so the user sees IMA's own message. if env.Code != 0 { if env.Code == 110030 { return fmt.Errorf("%w: ima code=%d msg=%s", @@ -294,6 +302,13 @@ func (c *client) DownloadURL(ctx context.Context, u urlInfo) ([]byte, string, er if u.URL == "" { return nil, "", fmt.Errorf("empty url") } + // The URL comes from an API response, so it is attacker-influenced as far + // as this process is concerned. downloadClient already guards the dial and + // every redirect hop; this rejects an unsafe target before the first + // connection is even attempted. + if err := utils.ValidateURLForSSRF(u.URL); err != nil { + return nil, "", fmt.Errorf("media URL rejected: %w", err) + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.URL, nil) if err != nil { return nil, "", fmt.Errorf("create download request: %w", err) @@ -307,7 +322,7 @@ func (c *client) DownloadURL(ctx context.Context, u urlInfo) ([]byte, string, er if err != nil { return nil, "", fmt.Errorf("download: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, "", fmt.Errorf("download http error: status=%d", resp.StatusCode) diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.go index b21a9c82..dd579544 100644 --- a/internal/datasource/connector/ima/connector.go +++ b/internal/datasource/connector/ima/connector.go @@ -42,10 +42,11 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig return nil } -// ResolveResourceAncestors: IMA knowledge bases are exposed as a flat list of -// top-level resources, so a lazy-loaded picker has no ancestors to reveal. +// ResolveResourceAncestors returns nothing: IMA knowledge bases are exposed as +// a flat list of top-level resources, so a lazy-loaded picker has no ancestors +// to reveal. func (c *Connector) ResolveResourceAncestors( - ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, + _ context.Context, _ *types.DataSourceConfig, _ []string, ) ([]string, error) { return []string{}, nil } @@ -103,7 +104,7 @@ func (c *Connector) ListResources( return nil, fmt.Errorf("search_knowledge_base fallback: %w", err) } for _, b := range resp.InfoList { - bases = append(bases, kbLite{ID: b.ID, Name: b.Name, CoverURL: b.CoverURL}) + bases = append(bases, kbLite(b)) } if resp.IsEnd || resp.NextCursor == "" { break @@ -237,7 +238,6 @@ func (c *Connector) walk( newCursor := &imaCursor{ LastSyncTime: time.Now(), KBLogical: make(map[string]map[string]string), - KBMedia: make(map[string]map[string]string), } var out []types.FetchedItem @@ -247,24 +247,22 @@ func (c *Connector) walk( return nil, nil, fmt.Errorf("list KB %s: %w", kbID, err) } - currentLogical := make(map[string]string, len(files)) - currentMedia := make(map[string]string, len(files)) + // present holds every logical_key IMA listed this cycle; it drives + // delete detection. syncedLogical holds only the keys we actually + // resolved this cycle and is what gets persisted as the cursor, so a + // transient failure is retried next run rather than being remembered + // as "unchanged" forever. + present := make(map[string]struct{}, len(files)) + syncedLogical := make(map[string]string, len(files)) keys := make([]string, len(files)) for i, f := range files { - // f.MediaType is 0 in list responses (IMA omits it there); we rely on - // (kb_id, parent_folder_id, title) which IMA enforces uniqueness on - // via check_repeated_names. Encoding media_type as 0 is fine as long - // as it stays consistent across syncs — which it does, since 0 is a - // constant here. - key := logicalKey(kbID, f.ParentFolderID, f.Title, f.MediaType) + key := logicalKey(kbID, f.ParentFolderID, f.Title) keys[i] = key - currentLogical[key] = f.MediaID - currentMedia[f.MediaID] = f.Title + present[key] = struct{}{} } - newCursor.KBLogical[kbID] = currentLogical - newCursor.KBMedia[kbID] = currentMedia + newCursor.KBLogical[kbID] = syncedLogical - replaced := 0 + var replaced, failed int for i, f := range files { key := keys[i] @@ -273,28 +271,41 @@ func (c *Connector) walk( if prevSet, ok := prev.KBLogical[kbID]; ok { if prevMediaID, seen := prevSet[key]; seen { if prevMediaID == f.MediaID { - continue // unchanged + // Unchanged: carry the entry forward so it is not + // re-fetched next cycle either. + syncedLogical[key] = f.MediaID + continue } replaced++ // media_id changed → replacement, fall through to re-fetch } } } - item, ok := fetchOneMedia(ctx, cli, cfg, kbID, key, f, folderPath) - if !ok { - continue + item, outcome := fetchOneMedia(ctx, cli, kbID, key, f, folderPath) + switch outcome { + case fetchOK: + syncedLogical[key] = f.MediaID + out = append(out, item) + case fetchSkipped: + // Deterministic skip (unsupported media type, no URL): record it + // so we do not pay for get_media_info on every future sync. + syncedLogical[key] = f.MediaID + case fetchFailed: + // Transient: leave the key out of the cursor so the next run + // retries. `present` still contains it, so it is not mistaken + // for a deletion. + failed++ } - out = append(out, item) } - // Deletion detection (incremental only) — based on logical_key vanishing, - // so a same-name replacement (media_id churns but logical_key stays) - // never fires a spurious delete. + // Deletion detection (incremental only) — based on logical_key vanishing + // from the listing, so neither a same-name replacement (media_id churns + // but logical_key stays) nor a failed download fires a spurious delete. deleted := 0 if incremental && prev != nil && prev.KBLogical != nil { if prevSet, ok := prev.KBLogical[kbID]; ok { for prevKey := range prevSet { - if _, still := currentLogical[prevKey]; !still { + if _, still := present[prevKey]; !still { out = append(out, types.FetchedItem{ ExternalID: prevKey, IsDeleted: true, @@ -306,8 +317,8 @@ func (c *Connector) walk( } } - logger.Infof(ctx, "[IMA] KB %s: total=%d replaced=%d deleted=%d", - kbID, len(files), replaced, deleted) + logger.Infof(ctx, "[IMA] KB %s: total=%d replaced=%d failed=%d deleted=%d", + kbID, len(files), replaced, failed, deleted) } if !incremental { @@ -396,50 +407,83 @@ func listAllKBFiles( return out, folderPath, nil } +// fetchOutcome distinguishes the three ways fetchOneMedia can end, because the +// caller has to treat them differently when building the sync cursor: only a +// transient failure must stay out of the cursor so it is retried next run. +type fetchOutcome int + +const ( + // fetchOK: the item was resolved and should be ingested. + fetchOK fetchOutcome = iota + // fetchSkipped: IMA cannot give us this item's content and never will + // (unsupported media type, no URL). Deterministic, so it is safe to + // remember in the cursor and stop re-probing it. + fetchSkipped + // fetchFailed: a transient error (API call or download). Must be retried. + fetchFailed +) + // fetchOneMedia calls get_media_info for a single item and, when possible, -// downloads its body. Returns false if the item should be skipped (unsupported -// media type, download error, note-type, etc.). +// downloads its body. // // externalID is the stable logical_key computed by the caller — same value // across syncs even after an IMA same-name replacement mints a new media_id. // The raw media_id is still preserved in metadata for observability. func fetchOneMedia( - ctx context.Context, cli *client, cfg *Config, + ctx context.Context, cli *client, kbID string, externalID string, f walkedFile, folderPath map[string]string, -) (types.FetchedItem, bool) { +) (types.FetchedItem, fetchOutcome) { info, err := cli.GetMediaInfo(ctx, f.MediaID) if err != nil { - logger.Warnf(ctx, "[IMA] get_media_info(%s) failed, skipping: %v", f.MediaID, err) - return types.FetchedItem{}, false + logger.Warnf(ctx, "[IMA] get_media_info(%s) failed, will retry next sync: %v", f.MediaID, err) + return types.FetchedItem{}, fetchFailed } if isSkippableMediaType(info.MediaType) { logger.Infof(ctx, "[IMA] skip media %s (title=%q media_type=%d): unsupported by wiki OpenAPI", f.MediaID, f.Title, info.MediaType) - return types.FetchedItem{}, false + return types.FetchedItem{}, fetchSkipped } if info.URLInfo.URL == "" { logger.Warnf(ctx, "[IMA] media %s (title=%q) has no url_info.url, skipping", f.MediaID, f.Title) - return types.FetchedItem{}, false + return types.FetchedItem{}, fetchSkipped } ext := extensionForMediaType(info.MediaType) + + // A media type with no fixed extension (web page, WeChat article, ...) is + // normally handed to the ingest layer as a bare URL so WeKnora fetches it + // itself. That only works for publicly reachable links: when IMA attaches + // auth headers the URL points at IMA-hosted storage and WeKnora's own + // fetch — which cannot carry those headers — would be rejected. In that + // case download here, where the headers are available, and infer the + // extension from the response instead. if ext == "" { - return types.FetchedItem{ - ExternalID: externalID, - Title: f.Title, - URL: info.URLInfo.URL, - SourceResourceID: kbID, - UpdatedAt: time.Now(), - Metadata: baseMetadata(externalID, f, folderPath, info, kbID), - }, true + if len(info.URLInfo.Headers) == 0 { + return types.FetchedItem{ + ExternalID: externalID, + Title: f.Title, + URL: info.URLInfo.URL, + SourceResourceID: kbID, + UpdatedAt: time.Now(), + Metadata: baseMetadata(externalID, f, folderPath, info, kbID), + }, fetchOK + } + ext = "html" } body, ct, err := cli.DownloadURL(ctx, info.URLInfo) if err != nil { - logger.Warnf(ctx, "[IMA] download media %s (title=%q) failed, skipping: %v", f.MediaID, f.Title, err) - return types.FetchedItem{}, false + logger.Warnf(ctx, "[IMA] download media %s (title=%q) failed, will retry next sync: %v", + f.MediaID, f.Title, err) + return types.FetchedItem{}, fetchFailed + } + + // IMA labels every image media_type=9, so trust the response's own + // Content-Type over the media_type default when it names a real format. + if detected := extensionForContentType(ct); detected != "" { + ext = detected } if ct == "" || strings.HasPrefix(ct, "application/octet-stream") { ct = mimeForExtension(ext) @@ -460,14 +504,17 @@ func fetchOneMedia( UpdatedAt: time.Now(), SourceResourceID: kbID, Metadata: baseMetadata(externalID, f, folderPath, info, kbID), - }, true + }, fetchOK } // baseMetadata builds the metadata map preserved on every ingested item. // externalID (the caller's logical_key) is stored so operators can join the // stable identity back to the raw media_id — useful when debugging why a // same-name replacement showed up as an update instead of an add. -func baseMetadata(externalID string, f walkedFile, folderPath map[string]string, info *getMediaInfoResp, kbID string) map[string]string { +func baseMetadata( + externalID string, f walkedFile, folderPath map[string]string, + info *getMediaInfoResp, kbID string, +) map[string]string { m := map[string]string{ "channel": types.ChannelIMA, "media_id": f.MediaID, diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.go new file mode 100644 index 00000000..e724a77d --- /dev/null +++ b/internal/datasource/connector/ima/connector_test.go @@ -0,0 +1,404 @@ +package ima + +import ( + "context" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" +) + +func TestConnector_Type(t *testing.T) { + if got := NewConnector().Type(); got != types.ConnectorTypeIMA { + t.Errorf("Type() = %q, want %q", got, types.ConnectorTypeIMA) + } +} + +// TestFetchIncremental_TransientFailureIsRetried is the regression test for the +// cursor bug: the cursor used to be built from the raw listing before any +// content was fetched, so an item whose download failed was recorded as +// "seen with this media_id" and every later incremental sync skipped it as +// unchanged. The document was then silently missing from the KB forever. +func TestFetchIncremental_TransientFailureIsRetried(t *testing.T) { + f := newFakeIMA(t) + good := fakeFile{MediaID: "m-good", Title: "Good", MediaType: mediaTypeMarkdown, Body: "# ok"} + flaky := fakeFile{ + MediaID: "m-flaky", Title: "Flaky", MediaType: mediaTypeMarkdown, + Body: "# later", DownloadFails: true, + } + f.setKB("kb1", []fakeFile{good, flaky}) + + c := NewConnector() + cfg := f.config("kb1") + goodKey := logicalKey("kb1", "", "Good") + flakyKey := logicalKey("kb1", "", "Flaky") + + items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + if _, ok := findItem(items, flakyKey); ok { + t.Fatalf("failed download must not be emitted, got %s", describeItems(items)) + } + mustFindItem(t, items, goodKey) + + state := decodeCursor(t, cursor) + if _, recorded := state.KBLogical["kb1"][flakyKey]; recorded { + t.Fatal("a failed item must not be recorded in the cursor, otherwise it is never retried") + } + if state.KBLogical["kb1"][goodKey] != "m-good" { + t.Fatalf("successful item missing from cursor: %#v", state.KBLogical["kb1"]) + } + + // Second pass: the download now succeeds and the item must be re-attempted. + flaky.DownloadFails = false + f.setKB("kb1", []fakeFile{good, flaky}) + + items2, cursor2, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + retried := mustFindItem(t, items2, flakyKey) + if string(retried.Content) != "# later" { + t.Errorf("retried content = %q, want %q", retried.Content, "# later") + } + if _, ok := findItem(items2, goodKey); ok { + t.Errorf("unchanged item must be skipped on the second pass, got %s", describeItems(items2)) + } + if decodeCursor(t, cursor2).KBLogical["kb1"][flakyKey] != "m-flaky" { + t.Error("the retried item should now be recorded in the cursor") + } +} + +// TestFetchIncremental_FailureIsNotADeletion guards the other half of the fix: +// keeping a failed item out of the cursor must not make the next sync mistake +// it for a document that vanished from IMA. +func TestFetchIncremental_FailureIsNotADeletion(t *testing.T) { + f := newFakeIMA(t) + good := fakeFile{MediaID: "m-good", Title: "Good", MediaType: mediaTypeMarkdown, Body: "# ok"} + flaky := fakeFile{MediaID: "m-flaky", Title: "Flaky", MediaType: mediaTypeMarkdown, DownloadFails: true} + f.setKB("kb1", []fakeFile{good, flaky}) + + c := NewConnector() + cfg := f.config("kb1") + flakyKey := logicalKey("kb1", "", "Flaky") + + _, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + + // Still failing, still listed by IMA: no tombstone may be emitted. + items, _, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + for _, it := range items { + if it.IsDeleted && it.ExternalID == flakyKey { + t.Fatalf("a still-listed item that merely failed to download must not be tombstoned: %s", + describeItems(items)) + } + } +} + +func TestFetchIncremental_RemovedItemIsTombstoned(t *testing.T) { + f := newFakeIMA(t) + keep := fakeFile{MediaID: "m-keep", Title: "Keep", MediaType: mediaTypeMarkdown, Body: "# keep"} + drop := fakeFile{MediaID: "m-drop", Title: "Drop", MediaType: mediaTypeMarkdown, Body: "# drop"} + f.setKB("kb1", []fakeFile{keep, drop}) + + c := NewConnector() + cfg := f.config("kb1") + dropKey := logicalKey("kb1", "", "Drop") + + _, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + + f.setKB("kb1", []fakeFile{keep}) + items, _, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + + tombstone, ok := findItem(items, dropKey) + if !ok || !tombstone.IsDeleted { + t.Fatalf("removed item must be tombstoned, got %s", describeItems(items)) + } + if tombstone.SourceResourceID != "kb1" { + t.Errorf("tombstone SourceResourceID = %q, want kb1", tombstone.SourceResourceID) + } +} + +// TestFetchIncremental_SameNameReplacementKeepsExternalID covers IMA minting a +// new media_id when a file is replaced in place: the item must come back as an +// update under the same external id rather than a delete plus an insert. +func TestFetchIncremental_SameNameReplacementKeepsExternalID(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-v1", Title: "Doc", MediaType: mediaTypeMarkdown, Body: "v1"}}) + + c := NewConnector() + cfg := f.config("kb1") + key := logicalKey("kb1", "", "Doc") + + _, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + + f.setKB("kb1", []fakeFile{{MediaID: "m-v2", Title: "Doc", MediaType: mediaTypeMarkdown, Body: "v2"}}) + items, cursor2, err := c.FetchIncremental(context.Background(), cfg, cursor) + if err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + + updated := mustFindItem(t, items, key) + if updated.IsDeleted { + t.Fatal("a same-name replacement must be an update, not a delete") + } + if string(updated.Content) != "v2" { + t.Errorf("content = %q, want v2", updated.Content) + } + if updated.Metadata["media_id"] != "m-v2" { + t.Errorf("metadata media_id = %q, want m-v2", updated.Metadata["media_id"]) + } + for _, it := range items { + if it.IsDeleted { + t.Errorf("no tombstone expected, got %s", describeItems(items)) + } + } + if decodeCursor(t, cursor2).KBLogical["kb1"][key] != "m-v2" { + t.Error("cursor should track the new media_id") + } +} + +// TestFetchIncremental_UnsupportedTypeIsProbedOnce checks that a deterministic +// skip is remembered, so an IMA knowledge base full of notes does not re-pay +// for get_media_info on every scheduled sync. +func TestFetchIncremental_UnsupportedTypeIsProbedOnce(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-note", Title: "Note", MediaType: mediaTypeNote}}) + + c := NewConnector() + cfg := f.config("kb1") + + items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil) + if err != nil { + t.Fatalf("first FetchIncremental: %v", err) + } + if len(items) != 0 { + t.Fatalf("note must be skipped, got %s", describeItems(items)) + } + if f.callCount("get_media_info:m-note") != 1 { + t.Fatalf("get_media_info calls = %d, want 1", f.callCount("get_media_info:m-note")) + } + + if _, _, err := c.FetchIncremental(context.Background(), cfg, cursor); err != nil { + t.Fatalf("second FetchIncremental: %v", err) + } + if got := f.callCount("get_media_info:m-note"); got != 1 { + t.Errorf("get_media_info calls after second sync = %d, want 1 (skip should be cached)", got) + } +} + +// TestFetchAll_AuthenticatedURLIsDownloaded covers media types with no fixed +// extension. When IMA attaches auth headers the URL is IMA-hosted and WeKnora's +// own fetch could not authenticate, so the connector must download it here and +// forward the headers. +func TestFetchAll_AuthenticatedURLIsDownloaded(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{ + MediaID: "m-web", + Title: "Article", + MediaType: mediaTypeWeb, + Body: "body", + ContentType: "text/html", + URLHeaders: map[string]string{"X-Ima-Token": "secret"}, + }}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + item := mustFindItem(t, items, logicalKey("kb1", "", "Article")) + if string(item.Content) != "body" { + t.Errorf("content = %q, want the downloaded body", item.Content) + } + if !strings.HasSuffix(item.FileName, ".html") { + t.Errorf("FileName = %q, want an .html suffix", item.FileName) + } + if got := f.downloadHeaders["m-web"].Get("X-Ima-Token"); got != "secret" { + t.Errorf("auth header forwarded = %q, want %q", got, "secret") + } +} + +// TestFetchAll_PublicURLStaysURLOnly is the counterpart: without auth headers +// the link is publicly reachable, so it is handed to the ingest layer as a URL +// and WeKnora renders the live page instead of a snapshot. +func TestFetchAll_PublicURLStaysURLOnly(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-web", Title: "Article", MediaType: mediaTypeWeb, Body: "ignored"}}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + item := mustFindItem(t, items, logicalKey("kb1", "", "Article")) + if len(item.Content) != 0 { + t.Errorf("public URL must not be downloaded, got %d bytes", len(item.Content)) + } + if item.URL == "" { + t.Error("URL-only item must carry a URL") + } + if f.callCount("download:m-web") != 0 { + t.Error("no download should have been attempted") + } +} + +// TestFetchAll_ImageExtensionFollowsContentType guards against naming a JPEG +// ".png": IMA reports every image as media_type=9, so only the download's +// Content-Type reveals the real format, and the extension drives the ingest +// pipeline's file-type checks. +func TestFetchAll_ImageExtensionFollowsContentType(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{ + MediaID: "m-img", + Title: "Photo", + MediaType: mediaTypeImage, + Body: "\xff\xd8\xff\xe0jpeg", + ContentType: "image/jpeg", + }}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + item := mustFindItem(t, items, logicalKey("kb1", "", "Photo")) + if !strings.HasSuffix(item.FileName, ".jpg") { + t.Errorf("FileName = %q, want a .jpg suffix for an image/jpeg body", item.FileName) + } +} + +func TestFetchAll_SkipsItemWithoutURL(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", []fakeFile{{MediaID: "m-empty", Title: "Empty", MediaType: mediaTypePDF, NoURL: true}}) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + if len(items) != 0 { + t.Fatalf("item without a URL must be skipped, got %s", describeItems(items)) + } +} + +// TestFetchAll_ResolvesFolderPath checks the mixed file/folder listing walk. +func TestFetchAll_ResolvesFolderPath(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", + []fakeFile{ + {MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"}, + {MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"}, + }, + fakeFolder{FolderID: "f1", Name: "Outer"}, + fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"}, + ) + + items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"}) + if err != nil { + t.Fatalf("FetchAll: %v", err) + } + + deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep")) + if deep.Metadata["folder_path"] != "Outer/Inner" { + t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"]) + } + root := mustFindItem(t, items, logicalKey("kb1", "", "Root")) + if root.Metadata["folder_path"] != "" { + t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"]) + } +} + +func TestListResources_ReturnsAddableBases(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb-b", nil) + f.setKB("kb-a", nil) + + resources, err := NewConnector().ListResources(context.Background(), f.config(), "") + if err != nil { + t.Fatalf("ListResources: %v", err) + } + if len(resources) != 2 { + t.Fatalf("got %d resources, want 2", len(resources)) + } + if resources[0].ExternalID != "kb-a" || resources[1].ExternalID != "kb-b" { + t.Errorf("resources are not sorted by id: %+v", resources) + } + if resources[0].Description != "desc kb-a" { + t.Errorf("description not enriched from get_knowledge_base: %q", resources[0].Description) + } +} + +// TestListResources_FallsBackToSearch covers tenants whose credential exposes +// read-only scopes, where get_addable_knowledge_base_list comes back empty. +func TestListResources_FallsBackToSearch(t *testing.T) { + f := newFakeIMA(t) + f.searchBases = []searchedKnowledgeBaseInfo{{ID: "kb-x", Name: "X", CoverURL: "https://cover"}} + + resources, err := NewConnector().ListResources(context.Background(), f.config(), "") + if err != nil { + t.Fatalf("ListResources: %v", err) + } + if len(resources) != 1 || resources[0].ExternalID != "kb-x" { + t.Fatalf("expected the search fallback to surface kb-x, got %+v", resources) + } + if resources[0].Metadata["cover_url"] != "https://cover" { + t.Errorf("cover_url = %v, want https://cover", resources[0].Metadata["cover_url"]) + } +} + +func TestListResources_ChildrenAreEmpty(t *testing.T) { + f := newFakeIMA(t) + f.setKB("kb1", nil) + + resources, err := NewConnector().ListResources(context.Background(), f.config(), "kb1") + if err != nil { + t.Fatalf("ListResources: %v", err) + } + if len(resources) != 0 { + t.Errorf("knowledge bases have no children, got %+v", resources) + } +} + +func TestValidate_InvalidCredentials(t *testing.T) { + f := newFakeIMA(t) + cfg := f.config() + cfg.Credentials["client_id"] = "" + + err := NewConnector().Validate(context.Background(), cfg) + if err == nil { + t.Fatal("expected an error for a missing client_id") + } + if !strings.Contains(err.Error(), "client_id") { + t.Errorf("error = %v, want it to name the missing field", err) + } +} + +func TestFetchIncremental_RequiresResourceIDs(t *testing.T) { + f := newFakeIMA(t) + if _, _, err := NewConnector().FetchIncremental(context.Background(), f.config(), nil); err == nil { + t.Fatal("expected an error when no knowledge base is selected") + } +} + +func TestConnectorIsRegisteredInMetadata(t *testing.T) { + meta, ok := datasource.ConnectorMetadataRegistry[types.ConnectorTypeIMA] + if !ok { + t.Fatal("IMA connector missing from the metadata registry") + } + if meta.Type != types.ConnectorTypeIMA { + t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA) + } +} diff --git a/internal/datasource/connector/ima/fake_ima_test.go b/internal/datasource/connector/ima/fake_ima_test.go new file mode 100644 index 00000000..5183c0c3 --- /dev/null +++ b/internal/datasource/connector/ima/fake_ima_test.go @@ -0,0 +1,313 @@ +package ima + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1) +// are reachable. Production keeps the default strict SSRF policy. +func TestMain(m *testing.M) { + _ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + secutils.ResetSSRFWhitelistForTest() + os.Exit(m.Run()) +} + +// fakeFile is one entry in a fake knowledge base listing. +type fakeFile struct { + MediaID string + Title string + ParentFolderID string + + // MediaType is what get_media_info reports for this file. + MediaType int32 + // Body is what the download URL serves. + Body string + // ContentType is the download response's Content-Type ("" omits the header). + ContentType string + // URLHeaders are the auth headers IMA attaches to url_info. + URLHeaders map[string]string + // NoURL makes get_media_info return an empty url_info.url. + NoURL bool + + // InfoFails makes get_media_info return a 500 for this media_id. + InfoFails bool + // DownloadFails makes the download URL return a 500. + DownloadFails bool +} + +// fakeFolder is a folder entry in a fake knowledge base listing. +type fakeFolder struct { + FolderID string + Name string + ParentFolderID string +} + +// fakeIMA is an in-process stand-in for the IMA OpenAPI. Only the endpoints +// the connector calls are implemented. +type fakeIMA struct { + server *httptest.Server + + mu sync.Mutex + // files and folders are keyed by knowledge base id, then by parent folder + // id ("" for the KB root). + files map[string]map[string][]fakeFile + folders map[string]map[string][]fakeFolder + // bases is what get_addable_knowledge_base_list returns. + bases []addableKnowledgeBaseInfo + // searchBases is what search_knowledge_base returns. + searchBases []searchedKnowledgeBaseInfo + + // calls counts requests per action, plus "download:". + calls map[string]int + // downloadHeaders records the headers seen by the last download of a media. + downloadHeaders map[string]http.Header +} + +func newFakeIMA(t *testing.T) *fakeIMA { + t.Helper() + f := &fakeIMA{ + files: map[string]map[string][]fakeFile{}, + folders: map[string]map[string][]fakeFolder{}, + calls: map[string]int{}, + downloadHeaders: map[string]http.Header{}, + } + mux := http.NewServeMux() + mux.HandleFunc(apiBasePath+"/", f.handleAPI) + mux.HandleFunc("/dl/", f.handleDownload) + f.server = httptest.NewServer(mux) + t.Cleanup(f.server.Close) + return f +} + +// setKB replaces the listing of a knowledge base and registers it as addable. +func (f *fakeIMA) setKB(kbID string, files []fakeFile, folders ...fakeFolder) { + f.mu.Lock() + defer f.mu.Unlock() + + byParent := map[string][]fakeFile{} + for _, file := range files { + byParent[file.ParentFolderID] = append(byParent[file.ParentFolderID], file) + } + f.files[kbID] = byParent + + foldersByParent := map[string][]fakeFolder{} + for _, folder := range folders { + foldersByParent[folder.ParentFolderID] = append(foldersByParent[folder.ParentFolderID], folder) + } + f.folders[kbID] = foldersByParent + + for _, b := range f.bases { + if b.ID == kbID { + return + } + } + f.bases = append(f.bases, addableKnowledgeBaseInfo{ID: kbID, Name: "KB " + kbID}) +} + +func (f *fakeIMA) callCount(key string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls[key] +} + +func (f *fakeIMA) record(key string) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls[key]++ +} + +// findFile locates a file by media_id across every knowledge base. +func (f *fakeIMA) findFile(mediaID string) (fakeFile, bool) { + f.mu.Lock() + defer f.mu.Unlock() + for _, byParent := range f.files { + for _, files := range byParent { + for _, file := range files { + if file.MediaID == mediaID { + return file, true + } + } + } + } + return fakeFile{}, false +} + +func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) { + action := strings.TrimPrefix(r.URL.Path, apiBasePath+"/") + f.record(action) + + var req map[string]interface{} + _ = json.NewDecoder(r.Body).Decode(&req) + str := func(k string) string { + if v, ok := req[k].(string); ok { + return v + } + return "" + } + + switch action { + case "get_addable_knowledge_base_list": + f.mu.Lock() + resp := getAddableKnowledgeBaseListResp{AddableKnowledgeBaseList: f.bases, IsEnd: true} + f.mu.Unlock() + writeEnvelope(w, 0, "", resp) + + case "search_knowledge_base": + f.mu.Lock() + resp := searchKnowledgeBaseResp{InfoList: f.searchBases, IsEnd: true} + f.mu.Unlock() + writeEnvelope(w, 0, "", resp) + + case "get_knowledge_base": + infos := map[string]knowledgeBaseInfo{} + if ids, ok := req["ids"].([]interface{}); ok { + for _, raw := range ids { + id, _ := raw.(string) + infos[id] = knowledgeBaseInfo{ID: id, Name: "KB " + id, Description: "desc " + id} + } + } + writeEnvelope(w, 0, "", getKnowledgeBaseResp{Infos: infos}) + + case "get_knowledge_list": + kbID, folderID := str("knowledge_base_id"), str("folder_id") + f.mu.Lock() + files := f.files[kbID][folderID] + folders := f.folders[kbID][folderID] + f.mu.Unlock() + + var list []json.RawMessage + for _, folder := range folders { + b, _ := json.Marshal(folderInfo{ + FolderID: folder.FolderID, + Name: folder.Name, + ParentFolderID: folder.ParentFolderID, + }) + list = append(list, b) + } + for _, file := range files { + // IMA omits media_type in list responses. + b, _ := json.Marshal(map[string]interface{}{ + "media_id": file.MediaID, + "title": file.Title, + "parent_folder_id": file.ParentFolderID, + }) + list = append(list, b) + } + writeEnvelope(w, 0, "", getKnowledgeListResp{KnowledgeList: list, IsEnd: true}) + + case "get_media_info": + mediaID := str("media_id") + f.record("get_media_info:" + mediaID) + file, ok := f.findFile(mediaID) + if !ok { + writeEnvelope(w, 110001, "unknown media", nil) + return + } + if file.InfoFails { + w.WriteHeader(http.StatusInternalServerError) + return + } + resp := getMediaInfoResp{MediaType: file.MediaType} + if !file.NoURL { + resp.URLInfo = urlInfo{ + URL: f.server.URL + "/dl/" + mediaID, + Headers: file.URLHeaders, + } + } + writeEnvelope(w, 0, "", resp) + + default: + writeEnvelope(w, 110001, "unsupported action "+action, nil) + } +} + +func (f *fakeIMA) handleDownload(w http.ResponseWriter, r *http.Request) { + mediaID := strings.TrimPrefix(r.URL.Path, "/dl/") + f.record("download:" + mediaID) + + f.mu.Lock() + f.downloadHeaders[mediaID] = r.Header.Clone() + f.mu.Unlock() + + file, ok := f.findFile(mediaID) + if !ok || file.DownloadFails { + w.WriteHeader(http.StatusInternalServerError) + return + } + if file.ContentType != "" { + w.Header().Set("Content-Type", file.ContentType) + } + _, _ = w.Write([]byte(file.Body)) +} + +func writeEnvelope(w http.ResponseWriter, code int, msg string, data interface{}) { + raw, _ := json.Marshal(data) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(apiEnvelope{Code: code, Msg: msg, Data: raw}) +} + +func (f *fakeIMA) config(resourceIDs ...string) *types.DataSourceConfig { + return &types.DataSourceConfig{ + Type: types.ConnectorTypeIMA, + Credentials: map[string]interface{}{ + "client_id": "cid", + "api_key": "key", + "base_url": f.server.URL, + }, + ResourceIDs: resourceIDs, + } +} + +// decodeCursor converts the connector cursor back into its typed form. +func decodeCursor(t *testing.T, cur *types.SyncCursor) *imaCursor { + t.Helper() + if cur == nil { + t.Fatal("cursor is nil") + } + b, err := json.Marshal(cur.ConnectorCursor) + if err != nil { + t.Fatalf("marshal cursor: %v", err) + } + var out imaCursor + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("unmarshal cursor: %v", err) + } + return &out +} + +// findItem returns the fetched item with the given external id. +func findItem(items []types.FetchedItem, externalID string) (types.FetchedItem, bool) { + for _, it := range items { + if it.ExternalID == externalID { + return it, true + } + } + return types.FetchedItem{}, false +} + +func mustFindItem(t *testing.T, items []types.FetchedItem, externalID string) types.FetchedItem { + t.Helper() + it, ok := findItem(items, externalID) + if !ok { + t.Fatalf("item %s not found in %s", externalID, describeItems(items)) + } + return it +} + +func describeItems(items []types.FetchedItem) string { + var parts []string + for _, it := range items { + parts = append(parts, fmt.Sprintf("{id=%s title=%q deleted=%t}", it.ExternalID, it.Title, it.IsDeleted)) + } + return "[" + strings.Join(parts, " ") + "]" +} diff --git a/internal/datasource/connector/ima/types.go b/internal/datasource/connector/ima/types.go index 5c8feee1..7c3f3298 100644 --- a/internal/datasource/connector/ima/types.go +++ b/internal/datasource/connector/ima/types.go @@ -202,21 +202,22 @@ type getMediaInfoResp struct { // imaCursor tracks per-knowledge-base state seen during the last sync. // -// Two parallel maps are kept, both keyed by kb_id: +// KBLogical: { logical_key: media_id } — the authoritative source for both +// delete detection AND "replacement" detection. IMA reassigns media_id +// whenever a same-named file is replaced in place, so a logical key (see +// logicalKey) is what we treat as a document's stable identity. +// A logical_key present last sync but absent this sync ⇒ delete. +// A logical_key whose media_id changed ⇒ replacement (re-fetch content; the +// emitted FetchedItem carries the same ExternalID so the ingest layer's +// "existing external_id → delete-and-recreate" path (see +// datasource_service.go applyFetchedItem) treats it as an update. // -// - KBLogical: { logical_key: media_id } — the authoritative source for -// both delete detection AND "replacement" detection. IMA reassigns -// media_id whenever a same-named file is replaced in place, so a logical -// key (see logicalKey) is what we treat as a document's stable identity. -// A logical_key present last sync but absent this sync ⇒ delete. -// A logical_key whose media_id changed ⇒ replacement (re-fetch content; -// the emitted FetchedItem carries the same ExternalID so the ingest -// layer's "existing external_id → delete-and-recreate" path (see -// datasource_service.go applyFetchedItem) treats it as an update. +// Only items that were successfully synced (or deterministically skipped) +// this cycle are recorded, so a transient download failure leaves the key +// absent and the next run retries it instead of treating it as unchanged. // -// - KBMedia (legacy): { media_id: title } — retained so cursors persisted -// by older versions still deserialize; not consulted for new detection. -// New cursors write this too, purely for observability in logs. +// KBMedia is a legacy field retained only so cursors persisted by earlier +// builds still deserialize; it is never written or consulted. type imaCursor struct { LastSyncTime time.Time `json:"last_sync_time"` KBLogical map[string]map[string]string `json:"kb_logical,omitempty"` @@ -226,8 +227,8 @@ type imaCursor struct { // logicalKey derives a stable identity for an IMA item that survives // same-name replacement (which mints a new media_id server-side). // -// The key is a short SHA-256 hex over (kb_id, parent_folder_id, title, -// media_type). Rationale: +// The key is a short SHA-256 hex over (kb_id, parent_folder_id, title). +// Rationale: // // - kb_id keeps identities isolated across knowledge bases (same file // copied into two KBs is two distinct docs from WeKnora's perspective). @@ -235,17 +236,20 @@ type imaCursor struct { // display name — the tuple IMA's own check_repeated_names uses to gate // uploads (see api.md), so IMA's own uniqueness constraint enforces // this key's uniqueness inside a KB. -// - media_type distinguishes "notes.md" (Markdown) from "notes.md" -// (Word) in the same folder — pathologically rare but cheap to guard. +// +// media_type is deliberately NOT part of the key: get_knowledge_list omits it +// (it only arrives later from get_media_info), so folding it in would mean +// every external_id shifts the day IMA starts populating the field in list +// responses — which reads as "every document deleted and re-added". // // Emitted as "ima_" + first 32 hex chars (128 bits) — small enough to fit // well within Postgres varchar(64) external_id columns, still >> collision // resistant for any realistic KB size. -func logicalKey(kbID, parentFolderID, title string, mediaType int32) string { +func logicalKey(kbID, parentFolderID, title string) string { h := sha256.New() // Delimiter can't appear in any component because it's neither a valid // IMA id character nor allowed in titles once we've folded to bytes. - fmt.Fprintf(h, "%s\x1f%s\x1f%s\x1f%d", kbID, parentFolderID, title, mediaType) + _, _ = fmt.Fprintf(h, "%s\x1f%s\x1f%s", kbID, parentFolderID, title) return "ima_" + hex.EncodeToString(h.Sum(nil))[:32] } @@ -301,6 +305,35 @@ func extensionForMediaType(t int32) string { } } +// extensionForContentType maps a response Content-Type back to a file +// extension, ignoring any "; charset=..." suffix. Returns "" for types we +// have no better guess for than the media_type-derived default. +// +// IMA reports every image as media_type=9 regardless of the real encoding, so +// the download's Content-Type is the only signal that distinguishes a JPEG +// from a PNG. Naming a JPEG ".png" breaks the extension-based file-type checks +// downstream in the ingest pipeline. +func extensionForContentType(contentType string) string { + mediaType := strings.ToLower(strings.TrimSpace(contentType)) + if i := strings.Index(mediaType, ";"); i >= 0 { + mediaType = strings.TrimSpace(mediaType[:i]) + } + switch mediaType { + case "image/png": + return "png" + case "image/jpeg", "image/jpg": + return "jpg" + case "image/gif": + return "gif" + case "image/webp": + return "webp" + case "image/bmp": + return "bmp" + default: + return "" + } +} + // mimeForExtension maps a file extension to a canonical MIME type. // Used when the downloaded URL response omits (or misreports) Content-Type. func mimeForExtension(ext string) string { @@ -331,6 +364,10 @@ func mimeForExtension(ext string) string { return "image/jpeg" case "webp": return "image/webp" + case "gif": + return "image/gif" + case "bmp": + return "image/bmp" case "epub": return "application/epub+zip" case "xmind": diff --git a/internal/datasource/connector/ima/types_test.go b/internal/datasource/connector/ima/types_test.go new file mode 100644 index 00000000..d325b275 --- /dev/null +++ b/internal/datasource/connector/ima/types_test.go @@ -0,0 +1,163 @@ +package ima + +import ( + "errors" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" +) + +func TestLogicalKey_IsStableAndScoped(t *testing.T) { + base := logicalKey("kb1", "f1", "Doc") + + if base != logicalKey("kb1", "f1", "Doc") { + t.Error("logicalKey must be deterministic") + } + if !strings.HasPrefix(base, "ima_") || len(base) != 36 { + t.Errorf("key = %q, want an ima_ prefix and 36 chars so it fits varchar(64)", base) + } + + for _, other := range []string{ + logicalKey("kb2", "f1", "Doc"), + logicalKey("kb1", "f2", "Doc"), + logicalKey("kb1", "f1", "Other"), + } { + if other == base { + t.Errorf("keys must differ across kb/folder/title, both were %q", base) + } + } +} + +// TestLogicalKey_DelimiterIsUnambiguous guards against a key collision from +// naively concatenating the components. +func TestLogicalKey_DelimiterIsUnambiguous(t *testing.T) { + if logicalKey("kb", "a", "bc") == logicalKey("kb", "ab", "c") { + t.Error("component boundaries must not be ambiguous") + } +} + +func TestParseIMAConfig(t *testing.T) { + tests := []struct { + name string + credentials map[string]interface{} + wantErr error + wantErrText string + }{ + { + name: "valid", + credentials: map[string]interface{}{"client_id": "cid", "api_key": "key"}, + }, + { + name: "missing client_id", + credentials: map[string]interface{}{"api_key": "key"}, + wantErr: datasource.ErrInvalidCredentials, + }, + { + name: "blank api_key", + credentials: map[string]interface{}{"client_id": "cid", "api_key": " "}, + wantErr: datasource.ErrInvalidCredentials, + }, + { + name: "loopback base_url rejected by SSRF policy", + credentials: map[string]interface{}{"client_id": "cid", "api_key": "key", "base_url": "http://169.254.169.254"}, + wantErrText: "SSRF", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseIMAConfig(&types.DataSourceConfig{Credentials: tt.credentials}) + switch { + case tt.wantErr != nil: + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + case tt.wantErrText != "": + if err == nil || !strings.Contains(err.Error(), tt.wantErrText) { + t.Fatalf("err = %v, want it to mention %q", err, tt.wantErrText) + } + default: + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + }) + } +} + +func TestParseIMAConfig_NilConfig(t *testing.T) { + if _, err := parseIMAConfig(nil); !errors.Is(err, datasource.ErrInvalidConfig) { + t.Fatalf("err = %v, want ErrInvalidConfig", err) + } +} + +func TestConfig_GetBaseURL(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"", DefaultBaseURL}, + {" ", DefaultBaseURL}, + {"https://ima.example.com/", "https://ima.example.com"}, + {"ima.example.com", "https://ima.example.com"}, + {"http://ima.example.com//", "http://ima.example.com"}, + } + for _, tt := range tests { + if got := (&Config{BaseURL: tt.in}).GetBaseURL(); got != tt.want { + t.Errorf("GetBaseURL(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestExtensionForContentType(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"image/jpeg", "jpg"}, + {"image/JPEG", "jpg"}, + {"image/png; charset=binary", "png"}, + {"image/webp", "webp"}, + {"application/pdf", ""}, + {"", ""}, + } + for _, tt := range tests { + if got := extensionForContentType(tt.in); got != tt.want { + t.Errorf("extensionForContentType(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestSanitizeFileName(t *testing.T) { + if got := sanitizeFileName(""); got != "untitled" { + t.Errorf("empty name = %q, want untitled", got) + } + if got := sanitizeFileName(`a/b\c:d*e?f"gi|j`); strings.ContainsAny(got, `/\:*?"<>|`) { + t.Errorf("filesystem-hostile characters survived: %q", got) + } + + // A long CJK title must be truncated on a rune boundary, not mid-sequence. + long := strings.Repeat("知识", 200) + got := sanitizeFileName(long) + if len(got) > 200 { + t.Errorf("len = %d bytes, want <= 200", len(got)) + } + if !strings.HasPrefix(long, got) { + t.Errorf("truncation corrupted the prefix: %q", got) + } +} + +func TestIsSkippableMediaType(t *testing.T) { + for _, mt := range []int32{mediaTypeNote, mediaTypeAISession, mediaTypeVideo} { + if !isSkippableMediaType(mt) { + t.Errorf("media type %d should be skippable", mt) + } + } + for _, mt := range []int32{mediaTypePDF, mediaTypeMarkdown, mediaTypeWeb} { + if isSkippableMediaType(mt) { + t.Errorf("media type %d should not be skippable", mt) + } + } +}