From e7c27ccc9e7fc806c583f6bb5de012cba12e5c53 Mon Sep 17 00:00:00 2001 From: jacentsao Date: Fri, 25 Sep 2026 22:23:48 +0800 Subject: [PATCH] feat(confluence): add directory sync and private image assets (#3319) * feat(confluence): support page subtree selection * feat(confluence): add private image asset resolver * fix(confluence): preserve pages during partial syncs * fix(confluence): cap total inlined image bytes per page Up to 30 images of 10 MB each could be inlined as base64, and every downstream conversion copies the page again, so one extreme page could peak above 1 GB. Cap the data URIs a page holds, in the download cache and the rewritten HTML, at 50 MB; images past the budget keep their original src like any other skipped image. --------- Co-authored-by: wizardchen Co-authored-by: lyingbug --- frontend/src/api/datasource/index.ts | 2 + frontend/src/i18n/locales/en-US.ts | 3 + frontend/src/i18n/locales/ja-JP.ts | 3 + frontend/src/i18n/locales/ko-KR.ts | 3 + frontend/src/i18n/locales/ru-RU.ts | 3 + frontend/src/i18n/locales/zh-CN.ts | 3 + .../settings/DataSourceEditorDialog.test.ts | 22 + .../settings/DataSourceEditorDialog.vue | 48 +- .../connector/confluence/asset_resolver.go | 301 ++++++ .../confluence/asset_resolver_test.go | 612 ++++++++++++ .../datasource/connector/confluence/client.go | 501 +++++++++- .../connector/confluence/client_pages_test.go | 65 ++ .../connector/confluence/connector.go | 503 +++++++--- .../confluence/connector_stream_test.go | 935 ++++++++++++++++-- .../connector/confluence/resource.go | 43 + .../connector/confluence/resource_test.go | 23 + .../connector/confluence/sync_plan.go | 112 +++ .../datasource/connector/confluence/types.go | 72 +- .../connector/confluence/types_test.go | 28 + 19 files changed, 3031 insertions(+), 251 deletions(-) create mode 100644 internal/datasource/connector/confluence/asset_resolver.go create mode 100644 internal/datasource/connector/confluence/asset_resolver_test.go create mode 100644 internal/datasource/connector/confluence/resource.go create mode 100644 internal/datasource/connector/confluence/resource_test.go create mode 100644 internal/datasource/connector/confluence/sync_plan.go diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts index 1ac7ac6d..19096307 100644 --- a/frontend/src/api/datasource/index.ts +++ b/frontend/src/api/datasource/index.ts @@ -82,6 +82,8 @@ export interface Resource { url: string parent_id?: string has_children?: boolean + /** Connector-specific hints, e.g. Confluence Cloud's top-level-container limitation. */ + metadata?: Record } // --- API calls --- diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 1c7e6dcf..681150b9 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -6979,6 +6979,9 @@ export default { authHeaders: 'Custom headers (optional)', authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud cannot list pages stored directly under top-level folders in this space; selecting the whole space still syncs them.' + }, comingSoon: 'Coming soon', docHint: 'Get credentials at:', openDoc: 'Open documentation', diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts index ac272ac6..308b3c0f 100644 --- a/frontend/src/i18n/locales/ja-JP.ts +++ b/frontend/src/i18n/locales/ja-JP.ts @@ -6979,6 +6979,9 @@ export default { authHeaders: 'カスタムヘッダー(任意)', authHeadersHint: '非公開フィード用です。1行に1つ「名前: 値」の形式で入力します(例: Authorization: Bearer xxxx)' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud では、スペース直下のフォルダーなどコンテナ配下のページをここに一覧できません。スペース全体を選択すれば同期されます。' + }, comingSoon: '近日対応予定', docHint: '認証情報の取得先:', openDoc: 'ドキュメントを開く', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index ead7f713..9bba3b96 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1125,6 +1125,9 @@ export default { authHeaders: '사용자 지정 헤더 (선택)', authHeadersHint: '비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud는 아직 스페이스 최상위 폴더 등 컨테이너 아래의 페이지를 여기에 나열할 수 없습니다. 스페이스 전체를 선택하면 함께 동기화됩니다.' + }, connectorDesc: { feishu: '페이슈 위키에서 문서, 스프레드시트, 파일 동기화', lark: 'Lark 위키에서 문서, 스프레드시트, 파일 동기화', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 4f075e0b..d5b8fb16 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1125,6 +1125,9 @@ export default { authHeaders: 'Пользовательские заголовки (необязательно)', authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud пока не может показать здесь страницы из папок верхнего уровня пространства; при выборе всего пространства они будут синхронизированы.' + }, connectorDesc: { feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki', lark: 'Синхронизация документов, таблиц и файлов из Lark Wiki', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 073f8649..5497981a 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1127,6 +1127,9 @@ export default { authHeaders: '自定义请求头(可选)', authHeadersHint: '用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud 暂不支持在此列出空间顶层文件夹等容器下的页面;选择整个空间仍会同步它们。' + }, connectorDesc: { feishu: '同步飞书知识库中的文档、表格、文件', lark: '同步 Lark 知识库中的文档、表格、文件(飞书国际版)', diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts index 43c46c76..f9f2d126 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts @@ -157,3 +157,25 @@ test('new GitLab data sources continue to test credentials without persistence', assert.deepEqual(f.calls.map(call => call.method), ['validateCredentials']) } finally { f.close() } }) + +test('Cloud hierarchy limitation stays visible after an empty space expansion', async () => { + const f = await fixture() + try { + f.vm.resources = [{ + external_id: 'space-1', + name: 'Cloud space', + type: 'space', + has_children: true, + metadata: { hierarchy_limitation: 'cloud_top_level_containers' }, + }] + f.vm.expandedResourceIds = new Set(['space-1']) + await nextTick() + assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true) + + f.vm.expandedResourceIds = new Set() + f.vm.loadedChildrenIds = new Set(['space-1']) + f.vm.resources = [{ ...f.vm.resources[0], has_children: false }] + await nextTick() + assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true) + } finally { f.close() } +}) diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue index 44b16aa4..cb60d2bb 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue @@ -449,6 +449,16 @@ async function ensureChildrenLoaded(id: string) { if (!existing.has(c.external_id)) merged.push(c) } resources.value = merged + } else { + // Connectors may conservatively advertise HasChildren to avoid an N+1 + // probe. Once lazy loading proves this is a leaf, collapse it and retain + // that fact so another expand cannot trigger another empty request. + resources.value = resources.value.map(r => r.external_id === id + ? { ...r, has_children: false } + : r) + const expanded = new Set(expandedResourceIds.value) + expanded.delete(id) + expandedResourceIds.value = expanded } loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id) } catch (e: any) { @@ -466,13 +476,21 @@ async function ensureChildrenLoaded(id: string) { const visibleTree = computed(() => { const roots = resources.value.filter(r => !r.parent_id) - const result: { resource: Resource; depth: number }[] = [] + const result: { resource: Resource; depth: number; noticeAfter?: boolean }[] = [] function walk(items: Resource[], depth: number) { for (const r of items) { result.push({ resource: r, depth }) if (r.has_children && expandedResourceIds.value.has(r.external_id)) { walk(childrenMap.value.get(r.external_id) || [], depth + 1) } + // Keep this visible after an empty root listing is recognized as a leaf. + if ( + r.metadata?.hierarchy_limitation === 'cloud_top_level_containers' && + (expandedResourceIds.value.has(r.external_id) || + (!r.has_children && loadedChildrenIds.value.has(r.external_id))) + ) { + result.push({ resource: r, depth: depth + 1, noticeAfter: true }) + } } } walk(roots, 0) @@ -1720,9 +1738,12 @@ const drawerConfirmText = computed(() => {
+
@@ -2576,6 +2605,19 @@ const drawerConfirmText = computed(() => { background: color-mix(in srgb, var(--td-text-color-placeholder) 8%, transparent); } +.resource-picker__notice { + --depth: 0; + margin: 0 0 2px; + padding: 4px 8px 4px calc(8px + var(--depth) * 14px); + font-size: var(--app-text-xs); + line-height: 1.5; + color: var(--td-text-color-placeholder); +} + +.resource-picker__notice:last-child { + margin-bottom: 0; +} + /* --- Step 2: empty state --- */ .ds-resource-empty { text-align: center; diff --git a/internal/datasource/connector/confluence/asset_resolver.go b/internal/datasource/connector/confluence/asset_resolver.go new file mode 100644 index 00000000..f8b06f60 --- /dev/null +++ b/internal/datasource/connector/confluence/asset_resolver.go @@ -0,0 +1,301 @@ +package confluence + +import ( + "context" + "encoding/base64" + "errors" + "html" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + + "github.com/Tencent/WeKnora/internal/logger" +) + +const ( + // maxPageImages is the total data URI image budget one page may carry into the + // downstream docparser, which enforces a single per-page limit (maxRemoteImages) + // across every data:image URI regardless of origin. The budget is shared: it + // covers data URI images already present in the page plus the ones the resolver + // generates, and a repeated image consumes it per occurrence, not per unique URL. + // Downloads stay deduplicated by URL, and only occurrences within the remaining + // budget are collected, so the connector never emits more data URIs than the + // pipeline will keep. + maxPageImages = 30 + // maxPageInlineBytes caps the total size of the data URIs one page may carry, + // both in the download cache and in the rewritten HTML. Without it an extreme + // page (maxPageImages occurrences of maxImageBytes images) would inline hundreds + // of MB of base64, copied again by every downstream conversion. Images past the + // budget keep their original src, like any other image the resolver skips. + maxPageInlineBytes = 50 << 20 + // imageDownloadConcurrency bounds in-flight image downloads per page so an + // image-heavy page cannot exhaust HTTP connections or spike memory. + imageDownloadConcurrency = 4 +) + +// errImagePageBudget marks an image dropped because the page's inline budget +// (maxPageInlineBytes) was already used up. +var errImagePageBudget = errors.New("confluence page image budget exhausted") + +// imgTagRe matches a full start tag, tolerating '>' inside quoted +// attribute values. The \b keeps it from matching . +var imgTagRe = regexp.MustCompile(`(?i)"']|"[^"]*"|'[^']*')*>`) + +// srcAttrRe captures the src attribute value. The leading \s and the value group +// keep it from matching data-src/srcset, and support double-quoted, +// single-quoted, or bare values. Group 1 is the "src=" prefix, group 2 the value. +var srcAttrRe = regexp.MustCompile(`(?i)(\ssrc\s*=\s*)("[^"]*"|'[^']*'|[^\s"'>]+)`) + +type imgAction uint8 + +const ( + imgKeep imgAction = iota // empty, data:, external, or non-http scheme: leave untouched + imgDownload // same-origin private candidate +) + +// assetResolver rewrites same-origin private Confluence images in a page's +// body.view HTML into inline data URIs. Persistence and resource:// rewriting +// are left entirely to the downstream docparser image pipeline; the resolver +// only performs the one step that pipeline cannot do on its own: an +// authenticated fetch of Confluence-private bytes. +type assetResolver struct { + client *client +} + +func newAssetResolver(c *client) *assetResolver { return &assetResolver{client: c} } + +// Resolve returns content with same-origin private occurrences replaced +// by data URIs, within the per-page budget that Resolve shares with any data URI +// images already in the page. It is best-effort: an image that is skipped, falls +// past the remaining budget, or fails to download keeps its original src, and image +// problems never fail the page. Downloads are deduplicated by URL, but the budget +// counts written data URIs (occurrences) to match the downstream pipeline, which +// caps every data:image URI per page regardless of where it came from. +func (r *assetResolver) Resolve(ctx context.Context, content string) string { + if r == nil || r.client == nil { + return content + } + // Data URI images already in the page consume the same downstream budget as the + // ones we generate, so reserve room for them before inlining any private image. + available := maxPageImages - countDataURIImages(content) + if available <= 0 { + return content + } + candidates := r.collect(content, available) + if len(candidates) == 0 { + return content + } + cache := r.downloadAll(ctx, candidates) + if len(cache) == 0 { + return content + } + inlined, inlinedBytes := 0, 0 + budgetSpent := false + return imgTagRe.ReplaceAllStringFunc(content, func(tag string) string { + if inlined >= available || budgetSpent { + return tag + } + src, ok := extractImageSrc(tag) + if !ok { + return tag + } + action, abs := r.classify(src) + if action != imgDownload { + return tag + } + dataURI := cache[abs] + if dataURI == "" { + return tag + } + if inlinedBytes+len(dataURI) > maxPageInlineBytes { + budgetSpent = true + return tag + } + inlined++ + inlinedBytes += len(dataURI) + return replaceImageSrc(tag, dataURI) + }) +} + +// collect returns the unique same-origin download candidates whose first +// occurrence falls within the remaining inline budget, in document order. Because +// Resolve writes at most budget data URIs, a URL first seen after that many +// occurrences can never be inlined, so it is not downloaded. Duplicates collapse to +// one entry, so a repeated image is fetched once yet may be inlined at every +// occurrence inside the budget. +func (r *assetResolver) collect(content string, budget int) []string { + tags := imgTagRe.FindAllString(content, -1) + order := make([]string, 0, len(tags)) + seen := make(map[string]struct{}, len(tags)) + occurrences := 0 + for _, tag := range tags { + src, ok := extractImageSrc(tag) + if !ok { + continue + } + action, abs := r.classify(src) + if action != imgDownload { + continue + } + if occurrences >= budget { + break + } + occurrences++ + if _, dup := seen[abs]; dup { + continue + } + seen[abs] = struct{}{} + order = append(order, abs) + } + return order +} + +// countDataURIImages returns how many tags in content already carry a +// data:image URI. The downstream docparser counts every data:image occurrence +// against the same per-page limit whether it came from the page or from us, so +// these must be reserved before the resolver inlines any private Confluence image. +func countDataURIImages(content string) int { + count := 0 + for _, tag := range imgTagRe.FindAllString(content, -1) { + src, ok := extractImageSrc(tag) + if !ok { + continue + } + if strings.HasPrefix(strings.ToLower(src), "data:image/") { + count++ + } + } + return count +} + +// downloadAll fetches each candidate with bounded concurrency and returns a +// URL-keyed cache of the successful ones as data URIs, holding at most +// maxPageInlineBytes of them. Failures and images past the budget are logged +// (redacted) and omitted, so the rewrite pass leaves their original src in place. +func (r *assetResolver) downloadAll(ctx context.Context, urls []string) map[string]string { + cache := make(map[string]string, len(urls)) + cachedBytes := 0 + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, imageDownloadConcurrency) + for _, u := range urls { + if ctx.Err() != nil { + break + } + wg.Add(1) + sem <- struct{}{} + go func(abs string) { + defer wg.Done() + defer func() { <-sem }() + data, mime, err := r.client.downloadImage(ctx, abs) + if err != nil { + logImageFailure(ctx, abs, err) + return + } + uri := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data) + mu.Lock() + fits := cachedBytes+len(uri) <= maxPageInlineBytes + if fits { + cache[abs] = uri + cachedBytes += len(uri) + } + mu.Unlock() + if !fits { + logImageFailure(ctx, abs, errImagePageBudget) + } + }(u) + } + wg.Wait() + return cache +} + +// classify decides whether src is a same-origin private candidate to download or +// should be left untouched, returning the normalized absolute URL for downloads. +func (r *assetResolver) classify(src string) (imgAction, string) { + s := strings.TrimSpace(src) + if s == "" || strings.HasPrefix(strings.ToLower(s), "data:") { + return imgKeep, "" + } + u, err := url.Parse(s) + if err != nil { + return imgKeep, "" + } + if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" { + // file:, javascript:, and any other scheme are never fetched. + return imgKeep, "" + } + if u.Host != "" { + if u.Scheme == "" { + // Protocol-relative //host/path: only same-origin is eligible. + base, berr := url.Parse(r.client.cfg.baseURL) + if berr != nil || !strings.EqualFold(base.Host, u.Host) { + return imgKeep, "" + } + } else if !r.client.sameOrigin(u) { + // Absolute external http(s): keep, never send Confluence credentials. + return imgKeep, "" + } + } + abs, err := r.client.resolveEndpoint(s) + if err != nil { + return imgKeep, "" + } + return imgDownload, abs +} + +// logImageFailure records a redacted warning: host and path only, never the +// query (which may carry signed tokens), the Authorization header, or the body. +func logImageFailure(ctx context.Context, abs string, err error) { + host, path := abs, "" + if u, perr := url.Parse(abs); perr == nil { + host, path = u.Host, u.Path + } + reason := "download_failed" + switch { + case errors.Is(err, errImageTooLarge): + reason = "oversize" + case errors.Is(err, errImageNonImageContent): + reason = "non_image_content_type" + case errors.Is(err, errImageRedirectOffOrigin): + reason = "cross_origin_redirect" + case errors.Is(err, errImagePageBudget): + reason = "page_budget" + } + var api *apiError + if errors.As(err, &api) { + reason = "status_" + strconv.Itoa(api.status) + } + logger.Warnf(ctx, "[Confluence] failed to resolve image host=%s path=%s reason=%s", host, path, reason) +} + +// extractImageSrc returns the HTML-unescaped src value of an tag. +func extractImageSrc(tag string) (string, bool) { + m := srcAttrRe.FindStringSubmatch(tag) + if m == nil { + return "", false + } + value := strings.TrimSpace(m[2]) + if len(value) >= 2 { + if q := value[0]; (q == '"' || q == '\'') && value[len(value)-1] == q { + value = value[1 : len(value)-1] + } + } + value = strings.TrimSpace(html.UnescapeString(value)) + if value == "" { + return "", false + } + return value, true +} + +// replaceImageSrc swaps only the src attribute value for a double-quoted data +// URI, preserving the rest of the tag byte-for-byte. +func replaceImageSrc(tag, dataURI string) string { + loc := srcAttrRe.FindStringSubmatchIndex(tag) + if loc == nil { + return tag + } + start, end := loc[4], loc[5] + return tag[:start] + `"` + dataURI + `"` + tag[end:] +} diff --git a/internal/datasource/connector/confluence/asset_resolver_test.go b/internal/datasource/connector/confluence/asset_resolver_test.go new file mode 100644 index 00000000..98b12483 --- /dev/null +++ b/internal/datasource/connector/confluence/asset_resolver_test.go @@ -0,0 +1,612 @@ +package confluence + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" +) + +const testBaseURL = "https://confluence.test/wiki" + +func newImageTestClient(rt roundTripper) *client { + return &client{ + cfg: config{baseURL: testBaseURL, username: "reader", secret: "secret"}, + http: &http.Client{Transport: rt}, + } +} + +func imageResp(status int, contentType string, body []byte) *http.Response { + header := make(http.Header) + if contentType != "" { + header.Set("Content-Type", contentType) + } + return &http.Response{ + StatusCode: status, + Header: header, + Body: io.NopCloser(bytes.NewReader(body)), + } +} + +func redirectResp(location string) *http.Response { + header := make(http.Header) + header.Set("Location", location) + return &http.Response{ + StatusCode: http.StatusFound, + Header: header, + Body: io.NopCloser(bytes.NewReader(nil)), + } +} + +// reqLog records outbound requests. Image downloads run concurrently, so every +// access is mutex-guarded. +type reqLog struct { + mu sync.Mutex + reqs []reqRecord +} + +type reqRecord struct { + host string + path string + hasAuth bool +} + +func newReqLog() *reqLog { return &reqLog{} } + +func (l *reqLog) add(req *http.Request) { + l.mu.Lock() + defer l.mu.Unlock() + l.reqs = append(l.reqs, reqRecord{ + host: req.URL.Host, + path: req.URL.Path, + hasAuth: req.Header.Get("Authorization") != "", + }) +} + +func (l *reqLog) count() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.reqs) +} + +func (l *reqLog) all() []reqRecord { + l.mu.Lock() + defer l.mu.Unlock() + return append([]reqRecord(nil), l.reqs...) +} + +func (l *reqLog) hasHost(host string) bool { + for _, r := range l.all() { + if r.host == host { + return true + } + } + return false +} + +func (l *reqLog) hasPath(path string) bool { + for _, r := range l.all() { + if r.path == path { + return true + } + } + return false +} + +func (l *reqLog) anyWithoutAuth() bool { + for _, r := range l.all() { + if !r.hasAuth { + return true + } + } + return false +} + +var testPNG = []byte("private-png-bytes") + +func testDataURI(body []byte) string { + return "data:image/png;base64," + base64.StdEncoding.EncodeToString(body) +} + +func TestAssetResolverKeepsPublicURL(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `

hi

` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("public URL was modified:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("public image was downloaded: %v", log.all()) + } +} + +func TestAssetResolverKeepsBase64(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("base64 image was modified:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("base64 image triggered a download: %v", log.all()) + } +} + +func TestAssetResolverKeepsNonHTTPScheme(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("non-http scheme was modified:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("non-http scheme triggered a download: %v", log.all()) + } +} + +func TestAssetResolverInlinesRelativeConfluenceImage(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + want := testDataURI(testPNG) + if !strings.Contains(out, `src="`+want+`"`) { + t.Fatalf("relative image not inlined as data URI:\n%s", out) + } + if strings.Contains(out, "/wiki/download/attachments") { + t.Fatalf("original private URL survived:\n%s", out) + } + reqs := log.all() + if len(reqs) != 1 { + t.Fatalf("expected 1 download, got %d: %v", len(reqs), reqs) + } + if reqs[0].host != "confluence.test" || !reqs[0].hasAuth { + t.Fatalf("download missing credentials or wrong host: %+v", reqs[0]) + } +} + +func TestAssetResolverInlinesSameOriginPrivateImage(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `x` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if !strings.Contains(out, `src="`+testDataURI(testPNG)+`"`) { + t.Fatalf("same-origin image not inlined:\n%s", out) + } + // The rest of the tag must be preserved. + if !strings.Contains(out, `alt="x"`) { + t.Fatalf("sibling attribute lost:\n%s", out) + } + if log.count() != 1 { + t.Fatalf("expected 1 download, got %d", log.count()) + } +} + +func TestAssetResolverDoesNotSendCredentialsCrossOrigin(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + + `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if log.hasHost("cdn.example.com") { + t.Fatalf("external host received a request: %v", log.all()) + } + if !strings.Contains(out, "https://cdn.example.com/a.png") { + t.Fatalf("external URL was modified:\n%s", out) + } + if !strings.Contains(out, testDataURI(testPNG)) { + t.Fatalf("same-origin image was not inlined:\n%s", out) + } + if log.count() != 1 || log.anyWithoutAuth() { + t.Fatalf("expected exactly one authenticated same-origin download: %v", log.all()) + } +} + +func TestAssetResolverRejectsCrossOriginRedirect(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + if req.URL.Host == "confluence.test" { + return redirectResp("https://evil.example.com/a.png"), nil + } + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("cross-origin redirect was followed and inlined:\n%s", out) + } + if log.hasHost("evil.example.com") { + t.Fatalf("credentials could have leaked to evil host: %v", log.all()) + } + if log.count() != 1 { + t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all()) + } +} + +func TestAssetResolverRejectsSameOriginRedirectOutsideContextPath(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + // The base URL is https://confluence.test/wiki. A redirect to another app on + // the same host stays same-origin but escapes the /wiki context path, and Go + // keeps sending Basic Auth on a same-host hop. It must be refused before the + // credentialed request is ever sent, not merely discarded afterward. + if strings.HasPrefix(req.URL.Path, "/wiki/") { + return redirectResp("https://confluence.test/other-app/leak.png"), nil + } + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("same-host redirect outside the context path was followed and inlined:\n%s", out) + } + if log.hasPath("/other-app/leak.png") { + t.Fatalf("credentials followed a redirect outside the context path: %v", log.all()) + } + if log.count() != 1 { + t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all()) + } +} + +func TestAssetResolverRejectsOversizedImage(t *testing.T) { + log := newReqLog() + big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1) + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", big), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("oversized image was inlined:\n%s", out) + } + if strings.Contains(out, "data:image") { + t.Fatalf("oversized image produced a data URI") + } +} + +func TestAssetResolverRejectsNonImageResponse(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "text/html; charset=utf-8", []byte("login required")), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("login HTML was inlined as an image:\n%s", out) + } +} + +func TestAssetResolverKeepsOriginalURLOnDownloadFailure(t *testing.T) { + for _, status := range []int{http.StatusForbidden, http.StatusNotFound, http.StatusInternalServerError} { + t.Run(http.StatusText(status), func(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(status, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("status %d modified the document:\n%s", status, out) + } + }) + } +} + +func TestAssetResolverDeduplicatesSamePageImages(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + src := `` + in := strings.Repeat(src, 5) + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if got := strings.Count(out, "data:image/png;base64,"); got != 5 { + t.Fatalf("expected 5 inlined copies, got %d:\n%s", got, out) + } + if log.count() != 1 { + t.Fatalf("duplicate src downloaded %d times, want 1", log.count()) + } +} + +func TestAssetResolverRespectsPerPageImageCap(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + var sb strings.Builder + total := maxPageImages + 5 + for i := 0; i < total; i++ { + sb.WriteString(``) + } + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String()) + + if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages { + t.Fatalf("inlined %d images, want cap %d", got, maxPageImages) + } + if log.count() != maxPageImages { + t.Fatalf("downloaded %d images, want cap %d", log.count(), maxPageImages) + } + if got := strings.Count(out, "/wiki/download/attachments"); got != 5 { + t.Fatalf("expected 5 images past the cap to keep original src, got %d", got) + } +} + +func TestAssetResolverCapsOccurrencesNotUniqueDownloads(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + // Two unique URLs but 35 occurrences. Counting the cap by unique URL would + // download cold.png too and inline 30 + 5 data URIs, overflowing the downstream + // per-page limit that counts every occurrence. The budget must be occurrences. + hot := `` + cold := `` + in := strings.Repeat(hot, maxPageImages) + strings.Repeat(cold, 5) + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages { + t.Fatalf("inlined %d data URIs, want cap %d", got, maxPageImages) + } + if got := strings.Count(out, "/wiki/download/attachments/1/cold.png"); got != 5 { + t.Fatalf("expected the 5 images past the cap to keep original src, got %d", got) + } + // The repeated image downloads once; the image past the budget never downloads. + if log.count() != 1 { + t.Fatalf("downloaded %d images, want 1 (dedup + no over-budget fetch): %v", log.count(), log.all()) + } + if log.hasPath("/wiki/download/attachments/1/cold.png") { + t.Fatalf("image past the occurrence cap was downloaded: %v", log.all()) + } +} + +// largeTestImage is just under maxImageBytes, so a handful of them overflow +// maxPageInlineBytes long before the per-page occurrence cap. +var largeTestImage = bytes.Repeat([]byte("a"), int(maxImageBytes)-1024) + +func TestAssetResolverStopsInliningPastPageByteBudget(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "image/png", largeTestImage), nil + }) + want := maxPageInlineBytes / len(testDataURI(largeTestImage)) + total := want + 3 + src := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), strings.Repeat(src, total)) + + if got := strings.Count(out, "data:image/png;base64,"); got != want { + t.Fatalf("inlined %d copies, want %d within the %d-byte page budget", got, want, maxPageInlineBytes) + } + if got := strings.Count(out, "/wiki/download/attachments/1/big.png"); got != total-want { + t.Fatalf("expected %d copies past the budget to keep original src, got %d", total-want, got) + } +} + +func TestAssetResolverDownloadCacheRespectsPageByteBudget(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "image/png", largeTestImage), nil + }) + want := maxPageInlineBytes / len(testDataURI(largeTestImage)) + urls := make([]string, 0, want+3) + for i := 0; i < want+3; i++ { + urls = append(urls, testBaseURL+"/download/attachments/1/img"+strings.Repeat("x", i+1)+".png") + } + cache := newAssetResolver(newImageTestClient(rt)).downloadAll(context.Background(), urls) + + if len(cache) != want { + t.Fatalf("cached %d images, want %d within the %d-byte page budget", len(cache), want, maxPageInlineBytes) + } +} + +func TestAssetResolverSharesCapWithExistingDataURIImages(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + // Data URI images already in the page share the downstream 30-image budget with + // the ones we generate. With 10 existing, only 20 of the 25 private images may + // be inlined; the rest keep their original URL instead of overflowing the cap. + var sb strings.Builder + for i := 0; i < 10; i++ { + sb.WriteString(``) + } + for i := 0; i < 25; i++ { + sb.WriteString(``) + } + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String()) + + // The 10 pre-existing data URIs are preserved untouched. + if got := strings.Count(out, "data:image/gif;base64,EXISTING"); got != 10 { + t.Fatalf("preserved %d existing data URIs, want 10", got) + } + // Exactly 20 private images inlined: 30 shared budget - 10 already present. + if got := strings.Count(out, testDataURI(testPNG)); got != 20 { + t.Fatalf("inlined %d private images, want 20", got) + } + // The 5 private images past the shared budget keep their original URL. + if got := strings.Count(out, "/wiki/download/attachments"); got != 5 { + t.Fatalf("expected 5 private images past the cap to keep original src, got %d", got) + } + // Only the 20 inlinable private images are downloaded. + if log.count() != 20 { + t.Fatalf("downloaded %d images, want 20: %v", log.count(), log.all()) + } +} + +func TestAssetResolverIgnoresDataSrcAttribute(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + // data-src must not be mistaken for src; there is no real src here. + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("data-src was treated as src:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("data-src triggered a download: %v", log.all()) + } +} + +func TestAssetResolverToleratesGreaterThanInAttribute(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `a > b` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if !strings.Contains(out, testDataURI(testPNG)) { + t.Fatalf("image with '>' in alt was not inlined:\n%s", out) + } + if !strings.Contains(out, `alt="a > b"`) { + t.Fatalf("alt attribute was corrupted:\n%s", out) + } +} + +func TestAssetResolverUnescapesAmpersandInQuery(t *testing.T) { + var gotPath string + var gotQuery string + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + gotPath = req.URL.Path + gotQuery = req.URL.RawQuery + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if !strings.Contains(out, "data:image/png;base64,") { + t.Fatalf("image with & query was not inlined:\n%s", out) + } + if gotPath != "/wiki/download/attachments/1/a.png" || gotQuery != "api=v2&v=3" { + t.Fatalf("query not unescaped: path=%q query=%q", gotPath, gotQuery) + } +} + +func TestDownloadPrivateImageLimitsBodySize(t *testing.T) { + big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1) + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "image/png", big), nil + }) + c := newImageTestClient(rt) + _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/big.png") + if !errors.Is(err, errImageTooLarge) { + t.Fatalf("downloadImage() error = %v, want errImageTooLarge", err) + } +} + +func TestDownloadPrivateImageValidatesContentType(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "application/json", []byte(`{"error":"unauthorized"}`)), nil + }) + c := newImageTestClient(rt) + _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png") + if !errors.Is(err, errImageNonImageContent) { + t.Fatalf("downloadImage() error = %v, want errImageNonImageContent", err) + } +} + +func TestDownloadPrivateImageSniffsWhenContentTypeMissing(t *testing.T) { + png := append([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, bytes.Repeat([]byte{0}, 64)...) + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "", png), nil + }) + c := newImageTestClient(rt) + body, mime, err := c.downloadImage( + context.Background(), + "https://confluence.test/wiki/download/attachments/1/a.png", + ) + if err != nil { + t.Fatalf("downloadImage() error = %v", err) + } + if mime != "image/png" { + t.Fatalf("sniffed mime = %q, want image/png", mime) + } + if !bytes.Equal(body, png) { + t.Fatalf("body mismatch: %d bytes", len(body)) + } +} + +func TestDownloadPrivateImageRejectsCrossOriginRedirect(t *testing.T) { + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + if req.URL.Host == "confluence.test" { + return redirectResp("https://evil.example.com/a.png"), nil + } + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + c := newImageTestClient(rt) + _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png") + if !errors.Is(err, errImageRedirectOffOrigin) { + t.Fatalf("downloadImage() error = %v, want errImageRedirectOffOrigin", err) + } +} + +// TestFetchStreamInlinesPrivateImageAsDataURI is the connector integration test: +// a body.view containing a same-origin private image flows through FetchStream +// and the emitted FetchedItem.Content carries an inline data URI (which the +// downstream docparser pipeline later persists and rewrites to resource://). +func TestFetchStreamInlinesPrivateImageAsDataURI(t *testing.T) { + api := &streamAPI{ + pages: []streamPage{{id: "p1", title: "Page", version: 1}}, + bodyHTML: map[string]string{"p1": `

hi

`}, + } + h := &captureHandler{} + if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), nil, h); err != nil { + t.Fatal(err) + } + if len(h.items) != 1 { + t.Fatalf("emitted %d items, want 1", len(h.items)) + } + content := string(h.items[0].Content) + if !strings.Contains(content, "data:image/png;base64,") { + t.Fatalf("private image was not inlined into Content:\n%s", content) + } + if strings.Contains(content, "/wiki/download/attachments") { + t.Fatalf("original private Confluence URL leaked into Content:\n%s", content) + } + if api.imageCalls != 1 { + t.Fatalf("image download calls = %d, want 1", api.imageCalls) + } +} diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go index ec156e38..66fa29ab 100644 --- a/internal/datasource/connector/confluence/client.go +++ b/internal/datasource/connector/confluence/client.go @@ -17,19 +17,29 @@ import ( ) const ( - maxJSONResponseBytes int64 = 20 << 20 - requestAttempts = 4 - maxErrorResponseRunes = 1000 - maxRetryDelay = 60 * time.Second - maxPaginationHops = 10000 + maxJSONResponseBytes int64 = 20 << 20 + maxImageBytes int64 = 10 << 20 + maxImageRedirects = 10 + requestAttempts = 4 + maxErrorResponseRunes = 1000 + maxRetryDelay = 60 * time.Second + maxPaginationHops = 10000 + maxTraversalNodes = 100000 + maxTransparentTraversalNodes = 10000 + maxTransparentTraversalDepth = 32 // maxEmptyServerPages bounds how far a Server/Data Center page listing - // follows _links.next through empty pages. Some builds keep returning - // next past the end of a space (#3596); permission filtering can also - // empty one page mid-listing, so a single empty page is not the end. + // follows _links.next through empty pages. maxEmptyServerPages = 3 - // maxDeletionProbes caps how many unlisted pages one sync looks up - // individually after a cut-short listing; the rest wait for the next run. - maxDeletionProbes = 200 + maxDeletionProbes = 200 +) + +var ( + errImageTooLarge = errors.New("confluence image exceeds size limit") + errImageNonImageContent = errors.New("confluence image response is not an image") + // errImageRedirectOffOrigin fires whenever a redirect target leaves the base + // URL boundary resolveEndpoint enforces — a different origin, or the same host + // but a path outside the configured context path (e.g. /wiki → /other-app). + errImageRedirectOffOrigin = errors.New("confluence image redirect left the configured base URL") ) type apiError struct { @@ -232,6 +242,121 @@ func (c *client) resourceURL(link string) string { return resolved } +// sameOrigin reports whether u shares scheme and host with the configured base +// URL. Only same-origin URLs may carry Confluence credentials. +func (c *client) sameOrigin(u *url.URL) bool { + if u == nil { + return false + } + base, err := url.Parse(c.cfg.baseURL) + if err != nil { + return false + } + return strings.EqualFold(base.Scheme, u.Scheme) && strings.EqualFold(base.Host, u.Host) +} + +// withinImageScope reports whether u stays inside the boundary resolveEndpoint +// enforces for the initial image URL: the same origin AND a path inside the +// configured context path. Redirect targets are validated with this same rule, +// not merely sameOrigin, because Go forwards Basic Auth on a same-host redirect — +// so a hop like /wiki/download → /other-app/image must fail closed too. +func (c *client) withinImageScope(u *url.URL) bool { + if u == nil { + return false + } + _, err := c.resolveEndpoint(u.String()) + return err == nil +} + +// downloadImage fetches a same-origin private image using the connector's +// credentials. It streams with a hard size cap, fails closed if a redirect left +// the Confluence origin, and validates the response really is an image. It is a +// best-effort single attempt: a failed image degrades to keeping its original +// URL rather than retrying or failing the page. +func (c *client) downloadImage(ctx context.Context, absURL string) ([]byte, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, absURL, nil) + if err != nil { + return nil, "", err + } + req.SetBasicAuth(c.cfg.username, c.cfg.secret) + req.Header.Set("Accept", "image/*,*/*;q=0.8") + resp, err := c.imageHTTPClient().Do(req) + if err != nil { + return nil, "", err + } + defer func() { _ = resp.Body.Close() }() + // Backstop for the redirect policy: imageHTTPClient's CheckRedirect already + // refuses any hop that leaves the base before it is followed, so this only + // re-verifies the final URL when the transport populated resp.Request. A custom + // RoundTripper may leave it nil, in which case there is no followed redirect to + // re-check and the download is allowed to proceed. + if resp.Request != nil && !c.withinImageScope(resp.Request.URL) { + return nil, "", errImageRedirectOffOrigin + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, "", &apiError{endpoint: "image", status: resp.StatusCode} + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxImageBytes+1)) + if err != nil { + return nil, "", err + } + if int64(len(body)) > maxImageBytes { + return nil, "", errImageTooLarge + } + mime, ok := imageContentType(resp.Header.Get("Content-Type"), body) + if !ok { + return nil, "", errImageNonImageContent + } + return body, mime, nil +} + +// imageHTTPClient reuses the connector's SSRF-safe transport and timeout but +// pins redirects to the configured Confluence base, so a private-image fetch can +// never be bounced to a third party or to another app on the same host. The +// redirect check fails closed: any hop that resolveEndpoint would reject — a +// foreign origin, or a same-host path outside the context path (which Go would +// still send Basic Auth to) — is refused outright rather than merely stripped of +// credentials. +func (c *client) imageHTTPClient() *http.Client { + return &http.Client{ + Transport: c.http.Transport, + Timeout: c.http.Timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= maxImageRedirects { + return errors.New("confluence image stopped after too many redirects") + } + if !c.withinImageScope(req.URL) { + return errImageRedirectOffOrigin + } + return nil + }, + } +} + +// imageContentType normalizes a response Content-Type to an image MIME type. An +// empty or generic binary type falls back to content sniffing; any explicit +// non-image type (for example the text/html login page Confluence returns when a +// token expired) is rejected so it is never stored as an image. +func imageContentType(header string, body []byte) (string, bool) { + ct := strings.ToLower(strings.TrimSpace(header)) + if i := strings.Index(ct, ";"); i >= 0 { + ct = strings.TrimSpace(ct[:i]) + } + switch { + case strings.HasPrefix(ct, "image/"): + return ct, true + case ct != "" && ct != "application/octet-stream": + return "", false + } + if sniff := strings.ToLower(http.DetectContentType(body)); strings.HasPrefix(sniff, "image/") { + if i := strings.Index(sniff, ";"); i >= 0 { + sniff = strings.TrimSpace(sniff[:i]) + } + return sniff, true + } + return "", false +} + func (c *client) ping(ctx context.Context) error { if c.cfg.cloud() { return c.get(ctx, "/api/v2/spaces?limit=1", nil) @@ -308,10 +433,6 @@ func withCloudPageQuery(next string) string { return parsed.String() } -// pages lists every current page in a space. complete is false when a Server -// listing was cut short after maxEmptyServerPages empty pages that still -// linked onward; the caller must not treat pages missing from such a listing -// as deleted. func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) { if c.cfg.cloud() { var all []page @@ -371,10 +492,6 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo return all, complete, err } -// pageInSpace reports whether a Server page is still a current page of the -// space. Confluence answers 404 for a page that was deleted or moved to the -// trash; a page that is no longer current or now belongs to another space is -// gone from this one too, matching what a complete listing would omit. func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) { var result struct { Status string `json:"status"` @@ -406,3 +523,349 @@ func (c *client) body(ctx context.Context, id string) (pageBody, error) { err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=body.view,version,space", &result) return result, err } + +// pageDetail obtains identity, ownership, and version without requiring a +// rendered body. Scope enumeration must remain complete when a body is +// temporarily unavailable, otherwise a transient content error would be +// indistinguishable from a missing page during deletion reconciliation. +func (c *client) pageDetail(ctx context.Context, id string) (page, error) { + var result page + if c.cfg.cloud() { + err := c.get(ctx, "/api/v2/pages/"+url.PathEscape(id), &result) + return result, err + } + err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=version,space", &result) + return result, err +} + +// validatePageOwnership performs the constant-cost check required before a +// picker expansion. Ancestors are deliberately not fetched here: they are only +// needed by ResolveResourceAncestors and buildSyncPlan. +func (c *client) validatePageOwnership(ctx context.Context, s space, pageID string) error { + full, err := c.pageDetail(ctx, pageID) + if err != nil { + return err + } + if c.cfg.cloud() { + if full.SpaceID != s.ID { + return fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID) + } + return nil + } + if full.Space.Key == "" || full.Space.Key != s.Key { + return fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + return nil +} + +func serverChildPagesEndpoint(pageID string) string { + query := url.Values{"expand": []string{serverPageExpand}, "limit": []string{"100"}} + return "/rest/api/content/" + url.PathEscape(pageID) + "/child/page?" + query.Encode() +} + +func serverTopLevelPagesEndpoint(spaceKey string) string { + query := url.Values{"depth": []string{"root"}, "expand": []string{serverPageExpand}, "limit": []string{"100"}} + return "/rest/api/space/" + url.PathEscape(spaceKey) + "/content/page?" + query.Encode() +} + +func (c *client) listServerPages(ctx context.Context, endpoint string) ([]page, error) { + start, _ := url.Parse(endpoint) + startQuery := start.Query() + var all []page + err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) { + var result pageList + if err := c.get(ctx, pageURL, &result); err != nil { + return "", err + } + all = append(all, result.Results...) + next := result.Links.Next + if next != "" { + next = withServerListQuery(next, startQuery) + } + return next, nil + }) + return all, err +} + +// Server pagination links commonly retain only start/limit. Restore the query +// that defines the listing scope (notably depth=root) as well as expand. +func withServerListQuery(next string, required url.Values) string { + parsed, err := url.Parse(next) + if err != nil { + return next + } + query := parsed.Query() + for key, values := range required { + if query.Get(key) == "" && len(values) > 0 { + query[key] = values + } + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +func (c *client) listCloudHierarchy(ctx context.Context, endpoint string) ([]page, error) { + var all []page + err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) { + var result cloudHierarchyList + if err := c.get(ctx, pageURL, &result); err != nil { + return "", err + } + for _, item := range result.Results { + all = append( + all, + page{ + ID: item.ID, + Title: item.Title, + Status: item.Status, + Kind: item.Type, + SpaceID: item.SpaceID, + Links: item.Links, + }, + ) + } + return result.Links.Next, nil + }) + return all, err +} + +// topLevelPages retrieves one level only. The Server/DC depth=root endpoint +// is paginated by the server; it deliberately never falls back to scanning all +// pages because that would defeat lazy selection for large spaces. +// +// Cloud applies the same lazy contract: depth=0 returns only root-level pages, +// and Cloud exposes no endpoint to enumerate a space's top-level folders +// (CONFCLOUD-84275), so pages nested directly under those containers stay +// unselectable in the picker for now. Recovering them would require a +// whole-space scan per expand, which is exactly what lazy loading forbids; +// selecting the whole space still syncs them. The connector flags this +// limitation through resource metadata so the picker can surface it. +func (c *client) topLevelPages(ctx context.Context, s space) ([]page, error) { + var pages []page + var err error + if c.cfg.cloud() { + pages, err = c.listCloudHierarchy( + ctx, + "/api/v2/spaces/"+url.PathEscape(s.ID)+"/pages?status=current&depth=0&limit=250", + ) + if err == nil { + visible := pages[:0] + for _, page := range pages { + if page.Kind == "" || page.Kind == "page" { + visible = append(visible, page) + } + } + pages = visible + } + } else { + pages, err = c.listServerPages(ctx, serverTopLevelPagesEndpoint(s.Key)) + } + if err == nil { + return pages, nil + } + // The depth=root navigation hint is Server/DC-specific; a Cloud 400/404 has + // different causes and must surface the raw API error for diagnosis. + if !c.cfg.cloud() { + var api *apiError + if errors.As(err, &api) && (api.status == http.StatusBadRequest || api.status == http.StatusNotFound) { + return nil, fmt.Errorf( + "confluence server does not support complete top-level page navigation for this space; "+ + "select the Space to sync all pages: %w", + err, + ) + } + } + return nil, err +} + +func (c *client) directChildPages(ctx context.Context, _ space, pageID string) ([]page, error) { + if c.cfg.cloud() { + return c.visibleCloudPageChildren(ctx, pageID) + } + return c.listServerPages(ctx, serverChildPagesEndpoint(pageID)) +} + +func cloudDirectChildrenEndpoint(kind, id string) (string, error) { + plural := map[string]string{ + "page": "pages", "folder": "folders", "database": "databases", "whiteboard": "whiteboards", "embed": "embeds", + }[kind] + if plural == "" { + return "", fmt.Errorf("unsupported Confluence Cloud hierarchy type %q", kind) + } + return "/api/v2/" + plural + "/" + url.PathEscape(id) + "/direct-children?limit=250", nil +} + +// visibleCloudPageChildren projects Cloud's mixed-content hierarchy onto a +// Page-only picker. Containers remain connector-internal and are traversed +// until the next visible Page is found. +func (c *client) visibleCloudPageChildren(ctx context.Context, pageID string) ([]page, error) { + return c.walkCloudPageHierarchy(ctx, pageID, false) +} + +// walkCloudPageHierarchy traverses Cloud's mixed content tree through +// direct-children endpoints. Picker expansion stops at visible pages, while a +// sync walk continues through them to return the complete page subtree. +func (c *client) walkCloudPageHierarchy(ctx context.Context, pageID string, descendPages bool) ([]page, error) { + type queuedNode struct { + id, kind string + depth int + } + queue := []queuedNode{{id: pageID, kind: "page"}} + visited := map[string]struct{}{"page:" + pageID: {}} + pages := make([]page, 0) + nodeLimit := maxTransparentTraversalNodes + if descendPages { + nodeLimit = maxTraversalNodes + } + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + endpoint, err := cloudDirectChildrenEndpoint(current.kind, current.id) + if err != nil { + return nil, err + } + children, err := c.listCloudHierarchy(ctx, endpoint) + if err != nil { + return nil, err + } + for _, child := range children { + kind := child.Kind + if kind == "" { + kind = "page" + } + key := kind + ":" + child.ID + if child.ID == "" || (kind == "page" && child.ID == pageID) { + continue + } + if _, exists := visited[key]; exists { + continue + } + if len(visited) >= nodeLimit { + return nil, fmt.Errorf( + "confluence Cloud hierarchy traversal exceeds %d nodes", + nodeLimit, + ) + } + visited[key] = struct{}{} + if kind == "page" { + pages = append(pages, child) + if !descendPages { + continue + } + } + depth := 0 + if kind != "page" { + depth = current.depth + 1 + if current.kind == "page" { + depth = 1 + } + } + if depth > maxTransparentTraversalDepth { + return nil, fmt.Errorf( + "confluence Cloud transparent traversal exceeds depth %d", + maxTransparentTraversalDepth, + ) + } + if _, err := cloudDirectChildrenEndpoint(kind, child.ID); err != nil { + return nil, err + } + queue = append(queue, queuedNode{id: child.ID, kind: kind, depth: depth}) + } + } + return pages, nil +} + +// pageAncestors validates that pageID belongs to s and returns its path from +// the space root. A selected page is fetched even if it is top-level, so a +// forged page:{space}:{page} ID cannot cross a space boundary. +func (c *client) pageAncestors(ctx context.Context, s space, pageID string) ([]page, error) { + if c.cfg.cloud() { + full, err := c.pageDetail(ctx, pageID) + if err != nil { + return nil, err + } + if full.SpaceID != s.ID { + return nil, fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID) + } + ancestors, err := c.listCloudHierarchy(ctx, "/api/v2/pages/"+url.PathEscape(pageID)+"/ancestors?limit=250") + if err != nil { + return nil, err + } + return ancestors, nil + } + var full page + if err := c.get(ctx, "/rest/api/content/"+url.PathEscape(pageID)+"?expand=ancestors,space", &full); err != nil { + return nil, err + } + if full.Space.Key == "" || full.Space.Key != s.Key { + return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + return full.Ancestors, nil +} + +// pageSubtree uses direct-child traversal for Server/DC because that endpoint +// is stable across supported versions. Cloud uses the same complete +// mixed-content hierarchy walk as the picker, but continues through visible +// pages to include every page in the selected subtree. +func (c *client) pageSubtree(ctx context.Context, s space, pageID string) ([]page, error) { + full, err := c.pageDetail(ctx, pageID) + if err != nil { + return nil, err + } + if c.cfg.cloud() { + if full.SpaceID != s.ID { + return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + } else if full.Space.Key == "" || full.Space.Key != s.Key { + return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + root := full + if c.cfg.cloud() { + descendants, err := c.walkCloudPageHierarchy(ctx, pageID, true) + if err != nil { + return nil, err + } + out := []page{root} + for _, descendant := range descendants { + // Direct-child responses are intentionally compact. Hydrate each page + // before version comparison; an unknown version must never be treated + // as an unchanged stable token. + detail, err := c.pageDetail(ctx, descendant.ID) + if err != nil { + return nil, err + } + if detail.SpaceID != s.ID { + return nil, fmt.Errorf("page %s does not belong to space %s", descendant.ID, s.ID) + } + out = append(out, detail) + } + return out, nil + } + queue, out := []page{root}, []page{root} + seen := map[string]bool{root.ID: true} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + children, err := c.directChildPages(ctx, s, current.ID) + if err != nil { + return nil, err + } + for _, child := range children { + if child.ID == "" || seen[child.ID] { + continue + } + if len(seen) >= maxTraversalNodes { + return nil, fmt.Errorf("confluence page subtree exceeds %d nodes", maxTraversalNodes) + } + seen[child.ID] = true + if child.SpaceID != "" && child.SpaceID != s.ID { + return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID) + } + if !c.cfg.cloud() && child.Space.Key != "" && child.Space.Key != s.Key { + return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID) + } + queue, out = append(queue, child), append(out, child) + } + } + return out, nil +} diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go index 1a7df281..1113a35d 100644 --- a/internal/datasource/connector/confluence/client_pages_test.go +++ b/internal/datasource/connector/confluence/client_pages_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -145,6 +146,70 @@ func TestServerSpacePageListPrefersFlatResults(t *testing.T) { } } +func TestTopLevelPagesServerRestoresScopeOnNext(t *testing.T) { + var depths, expands []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rest/api/space/ENG/content/page" { + http.NotFound(w, r) + return + } + depths = append(depths, r.URL.Query().Get("depth")) + expands = append(expands, r.URL.Query().Get("expand")) + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("start") == "100" { + _, _ = w.Write([]byte(`{ + "results": [{"id": "2", "title": "Second", "version": {"number": 2}}] + }`)) + return + } + _, _ = w.Write([]byte(`{ + "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}, + "results": [{"id": "1", "title": "Root", "version": {"number": 1}}] + }`)) + })) + t.Cleanup(server.Close) + + c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} + pages, err := c.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + if err != nil { + t.Fatal(err) + } + if len(pages) != 2 || pages[0].ID != "1" || pages[1].ID != "2" { + t.Fatalf("pages = %#v", pages) + } + // The server's _links.next drops depth and expand; the connector must + // rebuild the listing scope or later pages would look versionless. + if len(depths) != 2 || depths[0] != "root" || depths[1] != "root" { + t.Fatalf("depth query = %#v", depths) + } + if len(expands) != 2 || expands[0] != serverPageExpand || expands[1] != serverPageExpand { + t.Fatalf("expand query = %#v", expands) + } +} + +func TestTopLevelPagesCloudSurfacesRawAPIError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"space not found"}`)) + })) + t.Cleanup(server.Close) + + cloud := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()} + _, err := cloud.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + if err == nil { + t.Fatal("cloud topLevelPages unexpectedly succeeded") + } + if strings.Contains(err.Error(), "does not support") { + t.Fatalf("cloud 404 reused the Server/DC navigation hint: %v", err) + } + + srv := &client{cfg: config{baseURL: server.URL + "/wiki"}, http: server.Client()} + _, err = srv.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + if err == nil || !strings.Contains(err.Error(), "confluence server does not support") { + t.Fatalf("server error = %v; want the Server/DC navigation hint", err) + } +} + func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) { var depths []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go index fe2a6f5c..083d5bf1 100644 --- a/internal/datasource/connector/confluence/connector.go +++ b/internal/datasource/connector/confluence/connector.go @@ -3,10 +3,14 @@ package confluence import ( "context" + "crypto/sha256" "errors" "fmt" "net/http" + "sort" "strings" + "sync" + "time" htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/Tencent/WeKnora/internal/datasource" @@ -21,12 +25,64 @@ var ( // Connector implements datasource.StreamingConnector for Confluence. type Connector struct { - newClient func(config) (*client, error) + newClient func(config) (*client, error) + spaceCache spaceCache +} + +const pickerSpaceCacheTTL = time.Minute + +type spaceCache struct { + mu sync.Mutex + entries map[string]cachedSpaces +} + +type cachedSpaces struct { + values []space + expires time.Time } // NewConnector creates a Confluence connector. func NewConnector() *Connector { return &Connector{newClient: newClient} } +func (c *Connector) pickerSpaces(ctx context.Context, client *client, cfg config) ([]space, error) { + key := pickerSpaceCacheKey(cfg) + now := time.Now() + c.spaceCache.mu.Lock() + if cached, ok := c.spaceCache.entries[key]; ok && now.Before(cached.expires) { + values := append([]space(nil), cached.values...) + c.spaceCache.mu.Unlock() + return values, nil + } + c.spaceCache.mu.Unlock() + spaces, err := client.spaces(ctx) + if err != nil { + return nil, err + } + c.spaceCache.mu.Lock() + if c.spaceCache.entries == nil { + c.spaceCache.entries = make(map[string]cachedSpaces) + } + for key, cached := range c.spaceCache.entries { + if !now.Before(cached.expires) { + delete(c.spaceCache.entries, key) + } + } + c.spaceCache.entries[key] = cachedSpaces{ + values: append([]space(nil), spaces...), + expires: now.Add(pickerSpaceCacheTTL), + } + c.spaceCache.mu.Unlock() + return spaces, nil +} + +// pickerSpaceCacheKey partitions cached space visibility by every credential +// component that can affect permissions. The secret is fingerprinted so it +// never becomes a map key retained in process memory or appears in diagnostics. +func pickerSpaceCacheKey(cfg config) string { + fingerprint := sha256.Sum256([]byte(cfg.secret)) + return cfg.edition + "\x00" + cfg.baseURL + "\x00" + cfg.username + "\x00" + fmt.Sprintf("%x", fingerprint) +} + // Type returns the connector type identifier. func (*Connector) Type() string { return types.ConnectorTypeConfluence } @@ -52,13 +108,95 @@ func (c *Connector) Validate(ctx context.Context, ds *types.DataSourceConfig) er return client.ping(ctx) } -// ListResources returns the Confluence spaces the credentials can see. +// ListResources exposes a lazy Space → Page tree. It never scans a whole +// space merely to render the picker. func (c *Connector) ListResources( ctx context.Context, ds *types.DataSourceConfig, parentID string, ) ([]types.Resource, error) { - if parentID != "" { - return nil, nil + client, cfg, err := c.configured(ds) + if err != nil { + return nil, err } + if parentID == "" { + spaces, err := c.pickerSpaces(ctx, client, cfg) + if err != nil { + return nil, err + } + out := make([]types.Resource, 0, len(spaces)) + for _, s := range spaces { + metadata := map[string]interface{}{"space_key": s.Key} + if cfg.cloud() { + // Cloud cannot enumerate a space's top-level folders, so pages under + // them are unselectable in the picker; the frontend surfaces this + // limitation when the space is expanded. + metadata["hierarchy_limitation"] = "cloud_top_level_containers" + } + out = append(out, types.Resource{ + ExternalID: s.ID, Name: s.Name, Type: "space", + URL: client.resourceURL(s.Links.WebUI), HasChildren: true, + Metadata: metadata, + }) + } + return out, nil + } + ref, err := parseResourceID(parentID) + if err != nil { + return nil, err + } + spaces, err := c.pickerSpaces(ctx, client, cfg) + if err != nil { + return nil, err + } + var selected space + found := false + for _, s := range spaces { + if s.ID == ref.SpaceID { + selected, found = s, true + break + } + } + if !found { + return nil, fmt.Errorf("confluence space %s is unavailable", ref.SpaceID) + } + var pages []page + if ref.Kind == resourceSpace { + pages, err = client.topLevelPages(ctx, selected) + } else { + if ref.Kind == resourcePage { + if err := client.validatePageOwnership(ctx, selected, ref.PageID); err != nil { + return nil, fmt.Errorf("validate Confluence page %s: %w", ref.PageID, err) + } + } + pages, err = client.directChildPages(ctx, selected, ref.PageID) + } + if err != nil { + return nil, fmt.Errorf("list Confluence resources under %s: %w", parentID, err) + } + out := make([]types.Resource, 0, len(pages)) + for _, p := range pages { + out = append(out, pageResource(client, selected, parentID, p)) + } + return out, nil +} + +func pageResource(client *client, s space, parentID string, p page) types.Resource { + return types.Resource{ + ExternalID: makePageResourceID(s.ID, p.ID), ParentID: parentID, Name: p.Title, Type: "page", + URL: client.resourceURL(p.Links.WebUI), HasChildren: true, + Metadata: map[string]interface{}{"space_id": s.ID, "page_id": p.ID}, + } +} + +// ResolveResourceAncestors expands only the O(depth) path required to reveal +// persisted page selections in the lazy picker. Resolution is best effort: +// a deleted page, an unavailable space, or a malformed id skips just its own +// reveal path instead of failing every saved selection; the sync path keeps +// its strict validation in buildSyncPlan. +func (c *Connector) ResolveResourceAncestors( + ctx context.Context, + ds *types.DataSourceConfig, + resourceIDs []string, +) ([]string, error) { client, _, err := c.configured(ds) if err != nil { return nil, err @@ -67,25 +205,47 @@ func (c *Connector) ListResources( if err != nil { return nil, err } - out := make([]types.Resource, 0, len(spaces)) + byID := make(map[string]space, len(spaces)) for _, s := range spaces { - out = append(out, types.Resource{ - ExternalID: s.ID, - Name: s.Name, - Type: "space", - URL: client.resourceURL(s.Links.WebUI), - HasChildren: false, - Metadata: map[string]interface{}{"space_key": s.Key}, - }) + byID[s.ID] = s + } + seen, out := map[string]bool{}, []string{} + add := func(id string) { + if id != "" && !seen[id] { + seen[id] = true + out = append(out, id) + } + } + for _, id := range resourceIDs { + ref, err := parseResourceID(id) + if err != nil { + logger.Warnf(ctx, "skip malformed Confluence resource %q during ancestor resolution: %v", id, err) + continue + } + if ref.Kind == resourceSpace { + continue + } + s, ok := byID[ref.SpaceID] + if !ok { + logger.Warnf(ctx, "skip Confluence resource %s: space %s is unavailable", id, ref.SpaceID) + continue + } + ancestors, err := client.pageAncestors(ctx, s, ref.PageID) + if err != nil { + logger.Warnf(ctx, "skip Confluence resource %s during ancestor resolution: %v", id, err) + continue + } + add(s.ID) + for _, ancestor := range ancestors { + if ancestor.Kind != "" && ancestor.Kind != "page" { + continue + } + add(makePageResourceID(s.ID, ancestor.ID)) + } } return out, nil } -// ResolveResourceAncestors is a no-op: Confluence spaces are a flat list. -func (*Connector) ResolveResourceAncestors(context.Context, *types.DataSourceConfig, []string) ([]string, error) { - return nil, nil -} - // FetchAll syncs every selected space. Deletion reconciliation needs FetchStream. func (c *Connector) FetchAll( ctx context.Context, ds *types.DataSourceConfig, resourceIDs []string, @@ -158,50 +318,89 @@ func (c *Connector) fetchStream( if err != nil { return nil, err } - spaces, err := client.spaces(ctx) + plan, err := c.buildSyncPlan(ctx, client, ds.ResourceIDs) if err != nil { - return nil, fmt.Errorf("list Confluence spaces: %w", err) + return nil, err } - byID := make(map[string]space, len(spaces)) - for _, s := range spaces { - byID[s.ID] = s + currentRoots := make(map[string]bool, len(plan.Roots)) + for _, root := range plan.Roots { + currentRoots[root.ResourceID] = true } baseline, next := prepareSyncCursors(old, forceFull) - for _, resourceID := range ds.ResourceIDs { - s, found := byID[resourceID] - if !found { - return nil, fmt.Errorf( - "selected Confluence space %s is unavailable; refusing destructive reconciliation", - resourceID, - ) + // A prior successful body is reusable when ownership moves to a different + // selected root and this run cannot fetch the new body. Keep that version in + // the new root rather than dropping it during ownership reconciliation. + priorVersionByPage := make(map[string]string) + for _, pages := range baseline.SpacePages { + for id, version := range pages { + if _, exists := priorVersionByPage[id]; !exists { + priorVersionByPage[id] = version + } + } + } + // A page may be returned by multiple overlapping roots (for example during + // a configuration change). Fetch it once, while retaining a cursor entry + // per scope root so existing cursor JSON remains compatible. + seenPages := make(map[string]struct{}) + incompleteRoots := make(map[string]bool) + incompleteSpaces := make([]space, 0) + for _, root := range plan.Roots { + resourceID, s := root.ResourceID, root.Space + var pages []page + complete := true + if root.Kind == syncWholeSpace { + pages, complete, err = client.pages(ctx, s) + } else { + pages, err = client.pageSubtree(ctx, s, root.PageID) } - pages, complete, err := client.pages(ctx, s) if err != nil { - return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err) + return nil, fmt.Errorf("list pages in Confluence scope %s: %w", resourceID, err) } + priorPages := baseline.SpacePages[resourceID] if !complete { - logger.Warnf(ctx, - "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+ - "listed %d pages, checking unlisted ones one by one before deleting", - s.Key, maxEmptyServerPages, len(pages)) + logger.Warnf( + ctx, + "[Confluence] space %s listing was cut short; checking unlisted pages before deleting", + s.Key, + ) + incompleteRoots[resourceID] = true + incompleteSpaces = append(incompleteSpaces, s) } - priorPages, hadBaseline := baseline.SpacePages[resourceID] if next.SpacePages[resourceID] == nil { next.SpacePages[resourceID] = map[string]string{} } - seen := make(map[string]struct{}, len(pages)) + if complete && len(pages) == 0 && len(priorPages) > 0 { + return nil, fmt.Errorf( + "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline", + s.Key, len(priorPages), + ) + } for _, summary := range pages { if err := ctx.Err(); err != nil { return nil, err } - seen[summary.ID] = struct{}{} - version := pageVersion(summary) - if forceFull { - if next.SpacePages[resourceID][summary.ID] == version { + if _, duplicate := seenPages[summary.ID]; duplicate { + // The first scope already owns the emitted document; preserve this + // root's version record so a later selection change stays incremental. + if !forceFull { + if version, ok := priorPages[summary.ID]; ok { + next.SpacePages[resourceID][summary.ID] = version + } + } + continue + } + seenPages[summary.ID] = struct{}{} + version, versionKnown := pageVersion(summary) + // An unknown version cannot prove a page unchanged, so it must + // always refresh the body instead of skipping on a stable "t:". + if versionKnown { + if forceFull { + if next.SpacePages[resourceID][summary.ID] == version { + continue + } + } else if priorPages[summary.ID] == version { continue } - } else if priorPages[summary.ID] == version { - continue } full, err := client.body(ctx, summary.ID) if err != nil { @@ -211,13 +410,23 @@ func (c *Connector) fetchStream( if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil { return nil, emitErr } + if !forceFull { + if prior, ok := priorVersionByPage[summary.ID]; ok { + next.SpacePages[resourceID][summary.ID] = prior + } + } continue } - item, err := markdownItem(client, resourceID, summary, full) + item, err := markdownItem(ctx, client, resourceID, summary, full) if err != nil { if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil { return nil, emitErr } + if !forceFull { + if prior, ok := priorVersionByPage[summary.ID]; ok { + next.SpacePages[resourceID][summary.ID] = prior + } + } continue } if err := h.Emit(ctx, item); err != nil { @@ -228,101 +437,119 @@ func (c *Connector) fetchStream( return nil, err } } - if !complete { - if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil { - return nil, err - } - } else if hadBaseline { - if len(pages) == 0 && len(priorPages) > 0 { - return nil, fmt.Errorf( - "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline", - s.Key, len(priorPages), - ) - } - missing := 0 - for id := range priorPages { - if _, exists := seen[id]; !exists { - missing++ - } - } - if missing >= 20 && missing*100 >= len(priorPages)*80 { - return nil, fmt.Errorf( - "refusing Confluence mirror deletion in %s: %d/%d pages disappeared", - s.Key, missing, len(priorPages), - ) - } - for id := range priorPages { - if _, exists := seen[id]; exists { - continue - } - if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil { - return nil, err - } + } + // Reconcile previous owners only after every current scope has been enumerated. + // A page may move between roots, so a missing page must be checked against every + // scope whose listing was cut short before a tombstone can be emitted. + previousPages := make(map[string]string) + for owner, pages := range baseline.SpacePages { + for id := range pages { + if _, exists := previousPages[id]; !exists { + previousPages[id] = owner } } } + missing := make([]string, 0) + probeCount := 0 + for id := range previousPages { + if _, present := seenPages[id]; present { + continue + } + owner := previousPages[id] + for _, s := range incompleteSpaces { + if probeCount >= maxDeletionProbes { + return nil, fmt.Errorf( + "Confluence deletion reconciliation deferred: more than %d pages need verification", + maxDeletionProbes, + ) + } + probeCount++ + present, err := client.pageInSpace(ctx, id, s.Key) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf( + "Confluence deletion reconciliation deferred: could not verify page %s in space %s: %w", + id, s.Key, err, + ) + } + if present { + version := baseline.SpacePages[owner][id] + if next.SpacePages[s.ID] == nil { + next.SpacePages[s.ID] = make(map[string]string) + } + next.SpacePages[s.ID][id] = version + seenPages[id] = struct{}{} + break + } + } + if _, present := seenPages[id]; present { + continue + } + missing = append(missing, id) + } + // Protect still-selected roots from a suspiciously large shrink. Removed + // roots are an explicit scope change, so their out-of-scope pages can be + // tombstoned once every current scope has been enumerated safely. + owners := make([]string, 0, len(baseline.SpacePages)) + for owner := range baseline.SpacePages { + owners = append(owners, owner) + } + sort.Strings(owners) + for _, owner := range owners { + // A deliberate scope change may remove most of an old root. The guard is + // only for unexpected shrinkage of a still-selected, completely listed root. + if !currentRoots[owner] || incompleteRoots[owner] { + continue + } + pages := baseline.SpacePages[owner] + missingInRoot := 0 + for id := range pages { + if _, present := seenPages[id]; !present { + missingInRoot++ + } + } + if len(pages) > 0 && missingInRoot >= 20 && missingInRoot*100 >= len(pages)*80 { + return nil, fmt.Errorf( + "refusing Confluence mirror deletion: %d/%d previously synced pages would be removed from scope %s", + missingInRoot, len(pages), owner, + ) + } + } + sort.Strings(missing) + for _, id := range missing { + deleted := types.FetchedItem{ + ExternalID: id, IsDeleted: true, SourceResourceID: previousPages[id], + Metadata: map[string]string{"channel": types.ChannelConfluence}, + } + if err := h.Emit(ctx, deleted); err != nil { + return nil, err + } + // Drop the tombstoned page before checkpointing so the cursor records + // the deletion itself; an interrupted run resumes with the remaining + // candidates instead of re-emitting completed tombstones. + removePageOwnership(&next, id) + if err := h.Checkpoint(ctx, next.syncCursor()); err != nil { + return nil, err + } + } + // Commit exactly the current ownership roots only after reconciliation. This + // removes cancelled scopes while keeping the on-disk cursor field compatible. + committedRoots := make(map[string]map[string]string, len(plan.Roots)) + for _, root := range plan.Roots { + if pages := next.SpacePages[root.ResourceID]; pages != nil { + committedRoots[root.ResourceID] = pages + } else { + committedRoots[root.ResourceID] = map[string]string{} + } + } + next.SpacePages = committedRoots next.FullSync = false next.FullSyncBaseline = nil return next.syncCursor(), nil } -// reconcileUnlisted handles the pages a cut-short listing did not return. The -// listing proves nothing about them, so each one is looked up on its own and -// deleted only when Confluence confirms it is gone from the space. Pages that -// are still there, could not be checked, or exceed the per-run probe budget -// stay tracked with their prior version and are checked again next run. -func reconcileUnlisted( - ctx context.Context, - c *client, - h datasource.StreamHandler, - s space, - resourceID string, - priorPages map[string]string, - seen map[string]struct{}, - next *cursor, -) error { - probed := 0 - for id, version := range priorPages { - if _, exists := seen[id]; exists { - continue - } - if probed >= maxDeletionProbes { - next.SpacePages[resourceID][id] = version - continue - } - probed++ - present, err := c.pageInSpace(ctx, id, s.Key) - if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err) - next.SpacePages[resourceID][id] = version - continue - } - if present { - next.SpacePages[resourceID][id] = version - continue - } - if err := emitDeleted(ctx, h, next, resourceID, id); err != nil { - return err - } - } - return nil -} - -func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error { - deleted := types.FetchedItem{ - ExternalID: id, IsDeleted: true, SourceResourceID: resourceID, - Metadata: map[string]string{"channel": types.ChannelConfluence}, - } - if err := h.Emit(ctx, deleted); err != nil { - return err - } - delete(next.SpacePages[resourceID], id) - return h.Checkpoint(ctx, next.syncCursor()) -} - func failedPageItem(resourceID string, summary page, err error) types.FetchedItem { code, reason := classifyConfluenceError(err) return types.FetchedItem{ @@ -361,8 +588,18 @@ func classifyConfluenceError(err error) (code, reason string) { return "confluence_sync_failed", "Confluence page could not be synced; see server logs" } -func markdownItem(client *client, resourceID string, summary page, full pageBody) (types.FetchedItem, error) { +func markdownItem( + ctx context.Context, + client *client, + resourceID string, + summary page, + full pageBody, +) (types.FetchedItem, error) { html := full.Body.View.Value + // Inline same-origin private Confluence images as data URIs so the generic + // docparser image pipeline can persist them and rewrite to resource:// URLs. + // Best-effort: a failed image keeps its original src and never fails the page. + html = newAssetResolver(client).Resolve(ctx, html) markdown, err := htmltomd.ConvertString(html) if err != nil { return types.FetchedItem{}, err diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go index 2f38c787..0197a55d 100644 --- a/internal/datasource/connector/confluence/connector_stream_test.go +++ b/internal/datasource/connector/confluence/connector_stream_test.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" "testing" + "time" "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/types" @@ -26,19 +27,18 @@ type streamPage struct { } type streamAPI struct { - cloud bool - pages []streamPage - bodyCalls int - bodyStatus map[string]int - listCalls int - // endlessNext mimics Server/Data Center builds that keep linking to a - // next page after the last one, returning empty results forever (#3596). - endlessNext bool - // probe answers the per-page existence check (GET content/{id}?expand=space) - // for pages that are not in the listing: an HTTP status, or 200 with the - // space key the page now belongs to. - probe map[string]probeReply - probeCalls int + cloud bool + pages []streamPage + bodyCalls int + bodyStatus map[string]int + bodyHTML map[string]string + imageCalls int + listCalls int + ancestorCalls int + spaceCalls int + endlessNext bool + probe map[string]probeReply + probeCalls int } type probeReply struct { @@ -71,7 +71,17 @@ func (a *streamAPI) pageMaps() []any { func (a *streamAPI) response(req *http.Request) (*http.Response, error) { path := req.URL.Path switch { + case strings.HasPrefix(path, "/wiki/download/"): + a.imageCalls++ + header := make(http.Header) + header.Set("Content-Type", "image/png") + return &http.Response{ + StatusCode: http.StatusOK, + Header: header, + Body: io.NopCloser(bytes.NewReader([]byte("private-png-bytes"))), + }, nil case path == "/wiki/rest/api/space": + a.spaceCalls++ return a.jsonResponse(map[string]any{ "results": []any{map[string]any{ "id": 1, "key": "ENG", "name": "Engineering", @@ -80,19 +90,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { }) case path == "/wiki/rest/api/space/ENG/content/page": a.listCalls++ - if !a.endlessNext { + if req.URL.Query().Get("depth") == "root" { + return a.jsonResponse(map[string]any{"results": a.pageMaps()}) + } + if a.endlessNext { + results := a.pageMaps() + if a.listCalls > 1 { + results = []any{} + } return a.jsonResponse(map[string]any{ - "results": a.pageMaps(), + "results": results, + "_links": map[string]any{ + "next": fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100), + }, }) } - results := a.pageMaps() - if a.listCalls > 1 { - results = []any{} - } - next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100) return a.jsonResponse(map[string]any{ - "results": results, - "_links": map[string]any{"next": next}, + "results": a.pageMaps(), }) case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space": a.probeCalls++ @@ -108,10 +122,36 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { Body: io.NopCloser(strings.NewReader("{}")), }, nil } - return a.jsonResponse(map[string]any{ - "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}, - }) + return a.jsonResponse( + map[string]any{"id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}}, + ) + case strings.HasPrefix(path, "/wiki/rest/api/content/") && strings.HasSuffix(path, "/child/page"): + return a.jsonResponse(map[string]any{"results": []any{}}) case strings.HasPrefix(path, "/wiki/rest/api/content/"): + if req.URL.Query().Get("expand") == "ancestors,space" { + a.ancestorCalls++ + return a.jsonResponse(map[string]any{ + "id": strings.TrimPrefix(path, "/wiki/rest/api/content/"), + "space": map[string]any{"key": "ENG", "name": "Engineering"}, + "ancestors": []any{}, + }) + } + if req.URL.Query().Get("expand") == "version,space" { + id := strings.TrimPrefix(path, "/wiki/rest/api/content/") + for _, page := range a.pages { + if page.id == id { + return a.jsonResponse( + map[string]any{ + "id": page.id, + "title": page.title, + "version": map[string]any{"number": page.version}, + "space": map[string]any{"key": "ENG", "name": "Engineering"}, + }, + ) + } + } + return nil, errors.New("missing page") + } if req.URL.Query().Get("expand") != "body.view,version,space" { return nil, errors.New("page body did not request rendered body.view") } @@ -137,8 +177,14 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { }) } return a.jsonResponse(map[string]any{"results": pages}) + case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/ancestors"): + return a.jsonResponse(map[string]any{"results": []any{}}) + case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/descendants"): + return a.jsonResponse(map[string]any{"results": []any{}}) + case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/direct-children"): + return a.jsonResponse(map[string]any{"results": []any{}}) case strings.HasPrefix(path, "/wiki/api/v2/pages/"): - if req.URL.Query().Get("body-format") != "view" { + if req.URL.Query().Get("body-format") != "" && req.URL.Query().Get("body-format") != "view" { return nil, errors.New("cloud page body did not request body-format=view") } return a.pageBody(strings.TrimPrefix(path, "/wiki/api/v2/pages/")) @@ -158,12 +204,17 @@ func (a *streamAPI) pageBody(id string) (*http.Response, error) { } for _, page := range a.pages { if page.id == id { + value := "

" + page.title + "

" + if custom, ok := a.bodyHTML[id]; ok { + value = custom + } return a.jsonResponse(map[string]any{ "id": page.id, "title": page.title, + "spaceId": "1", "version": map[string]any{"number": page.version, "by": map[string]any{"displayName": "Ada"}}, "space": map[string]any{"key": "ENG", "name": "Engineering"}, "_links": map[string]any{"webui": "/wiki/pages/" + page.id}, - "body": map[string]any{"view": map[string]any{"value": "

" + page.title + "

"}}, + "body": map[string]any{"view": map[string]any{"value": value}}, }) } } @@ -196,10 +247,11 @@ func cloudStreamConfig() *types.DataSourceConfig { } type captureHandler struct { - items []types.FetchedItem - checkpoints []*types.SyncCursor - emitErr error - succeedFirst int + items []types.FetchedItem + checkpoints []*types.SyncCursor + emitErr error + succeedFirst int + checkpointErrAt int } func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error { @@ -214,6 +266,9 @@ func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error { } func (h *captureHandler) Checkpoint(_ context.Context, c *types.SyncCursor) error { + if h.checkpointErrAt > 0 && len(h.checkpoints)+1 == h.checkpointErrAt { + return errors.New("checkpoint failed") + } raw, _ := json.Marshal(c) var stored types.SyncCursor _ = json.Unmarshal(raw, &stored) @@ -248,16 +303,144 @@ func TestFetchStreamSkipsUnchangedPages(t *testing.T) { } } +func TestFetchStreamUnknownVersionDoesNotSkipBody(t *testing.T) { + // version.number=0 with no when/createdAt leaves the version unknown; the + // legacy stable "t:" cursor value must not suppress the body refresh. + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 0}}} + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "t:"}) + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if api.bodyCalls != 1 || len(h.items) != 1 || h.items[0].IsDeleted { + t.Fatalf("unknown-version page fetched=%d items=%#v", api.bodyCalls, h.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "" { + t.Fatalf("cursor stored a stable unknown version: %#v", decoded) + } +} + func TestFetchStreamDetectsDeletedPages(t *testing.T) { api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} h := &captureHandler{} old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) - if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h); err != nil { + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { t.Fatal(err) } if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" { t.Fatalf("deletion items = %#v", h.items) } + if decoded := decodeCursor(next); decoded.SpacePages["1"]["p2"] != "" { + t.Fatalf("deleted page stayed in cursor as a ghost: %#v", decoded) + } +} + +func TestFetchStreamRemovesDeletedPageFromCursor(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" { + t.Fatalf("deletion items = %#v", h.items) + } + decoded := decodeCursor(next) + if decoded.SpacePages["1"]["p1"] != "v:1" { + t.Fatalf("surviving page missing from cursor: %#v", decoded) + } + for _, checkpoint := range h.checkpoints { + if stored := decodeCursor(checkpoint); stored.SpacePages["1"]["p2"] != "" { + t.Fatalf("checkpoint recorded a not-yet-deleted page as deleted: %#v", stored) + } + } + // A follow-up run must not re-emit the already tombstoned page. + followUp := &captureHandler{} + if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), next, followUp); err != nil { + t.Fatal(err) + } + for _, item := range followUp.items { + if item.IsDeleted { + t.Fatalf("ghost page re-tombstoned on the next run: %#v", followUp.items) + } + } +} + +func TestFetchStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"}) + first := &captureHandler{succeedFirst: 1} + if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, first); err == nil { + t.Fatal("interrupted tombstone run unexpectedly succeeded") + } + if len(first.items) != 1 || !first.items[0].IsDeleted || first.items[0].ExternalID != "p2" { + t.Fatalf("first run items = %#v", first.items) + } + if len(first.checkpoints) != 1 { + t.Fatalf("first run checkpoints = %d; want 1", len(first.checkpoints)) + } + second := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), first.checkpoints[0], second) + if err != nil { + t.Fatal(err) + } + if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" { + t.Fatalf("resume must tombstone only the remaining page: %#v", second.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p3"] != "" { + t.Fatalf("resume cursor = %#v", decoded) + } +} + +func TestFetchFullStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"}) + // p1 re-fetch succeeds and the p2 tombstone is checkpointed before the + // worker dies on the p3 tombstone. + first := &captureHandler{succeedFirst: 2} + if _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first); err == nil { + t.Fatal("interrupted full-sync tombstone run unexpectedly succeeded") + } + if len(first.items) != 2 || first.items[1].ExternalID != "p2" || !first.items[1].IsDeleted { + t.Fatalf("first run items = %#v", first.items) + } + if len(first.checkpoints) != 2 { + t.Fatalf("first run checkpoints = %d; want 2", len(first.checkpoints)) + } + deletionCheckpoint := decodeCursor(first.checkpoints[1]) + if !deletionCheckpoint.FullSync { + t.Fatalf("mid-full-sync checkpoint lost the resume flag: %#v", deletionCheckpoint) + } + if baseline := deletionCheckpoint.FullSyncBaseline["1"]; baseline["p1"] != "v:1" || baseline["p2"] != "" || + baseline["p3"] != "v:1" { + t.Fatalf("tombstone checkpoint did not advance the deletion baseline: %#v", deletionCheckpoint) + } + + resumeAPI := &streamAPI{pages: api.pages} + second := &captureHandler{} + next, err := newStreamConnector( + resumeAPI, + ).FetchFullStream(context.Background(), streamConfig(), first.checkpoints[1], second) + if err != nil { + t.Fatal(err) + } + if resumeAPI.bodyCalls != 0 { + t.Fatalf("resume re-fetched %d already synced page bodies", resumeAPI.bodyCalls) + } + if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" { + t.Fatalf("resume must tombstone only the remaining page: %#v", second.items) + } + decoded := decodeCursor(next) + if decoded.FullSync || decoded.FullSyncBaseline != nil { + t.Fatalf("completed full sync left resume fields set: %#v", decoded) + } + if decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "" || + decoded.SpacePages["1"]["p3"] != "" { + t.Fatalf("completed resume cursor = %#v", decoded) + } } func TestFetchStreamRefusesMassDeletion(t *testing.T) { @@ -275,6 +458,67 @@ func TestFetchStreamRefusesMassDeletion(t *testing.T) { } } +func TestFetchStreamRefusesMassDeletionInSurvivingScope(t *testing.T) { + prior := make(map[string]string, 100) + for i := 0; i < 100; i++ { + prior[fmt.Sprintf("p%03d", i)] = "v:1" + } + // The selection is unchanged; a broken listing suddenly returns only five + // of the hundred previously synced pages. + listed := make([]streamPage, 0, 5) + for i := 0; i < 5; i++ { + listed = append(listed, streamPage{id: fmt.Sprintf("p%03d", i), title: "Page", version: 1}) + } + api := &streamAPI{pages: listed} + h := &captureHandler{} + _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h) + if err == nil || len(h.items) != 0 { + t.Fatalf("surviving-scope mass deletion err=%v items=%#v", err, h.items) + } +} + +func TestFetchStreamAllowsExplicitSpaceToSubtreeShrink(t *testing.T) { + prior := make(map[string]string, 100) + for i := 0; i < 100; i++ { + prior[fmt.Sprintf("p%03d", i)] = "v:1" + } + // The user deliberately narrows a whole-space selection down to a page + // subtree. The mass-deletion guard must only protect a surviving root. + api := &streamAPI{pages: []streamPage{{id: "erp", title: "ERP", version: 2}}} + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:erp"} + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, streamCursor(prior), h) + if err != nil { + t.Fatalf("explicit scope shrink: %v", err) + } + if len(h.items) != len(prior)+1 || h.items[0].ExternalID != "erp" { + t.Fatalf("scope shrink items = %d, want selected page plus %d tombstones", len(h.items), len(prior)) + } + deleted := 0 + for _, item := range h.items[1:] { + if !item.IsDeleted { + t.Fatalf("scope shrink emitted a non-tombstone outside the new scope: %#v", item) + } + deleted++ + } + if deleted != len(prior) { + t.Fatalf("scope shrink tombstones = %d, want %d", deleted, len(prior)) + } + decoded := decodeCursor(next) + if len(decoded.SpacePages) != 1 || decoded.SpacePages["page:1:erp"]["erp"] != "v:2" { + t.Fatalf("scope shrink cursor retained cancelled roots: %#v", decoded.SpacePages) + } + if _, oldRootPresent := decoded.SpacePages["1"]; oldRootPresent { + t.Fatalf("cancelled space root returned to cursor: %#v", decoded.SpacePages) + } + for _, item := range h.items[1:] { + if item.SourceResourceID != "1" { + t.Fatalf("tombstone source = %q, want old root 1", item.SourceResourceID) + } + } +} + func TestFetchStreamRefusesEmptyListingWhenBaselineExists(t *testing.T) { prior := map[string]string{"p1": "v:1", "p2": "v:1"} api := &streamAPI{} @@ -349,6 +593,50 @@ func TestFetchFullStreamResumesFromCheckpoint(t *testing.T) { } } +func TestFetchFullStreamRetryRefetchesFailedPage(t *testing.T) { + pages := []streamPage{ + {id: "p1", title: "One", version: 1}, + {id: "p2", title: "Broken then healthy", version: 2}, + {id: "p3", title: "Three", version: 1}, + } + api := &streamAPI{pages: pages, bodyStatus: map[string]int{"p2": http.StatusForbidden}} + first := &captureHandler{checkpointErrAt: 2} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"}) + _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first) + if err == nil { + t.Fatal("interrupted full sync unexpectedly succeeded") + } + if len(first.checkpoints) != 1 { + t.Fatalf("first full-sync checkpoints = %d; want one successful checkpoint", len(first.checkpoints)) + } + checkpoint := decodeCursor(first.checkpoints[0]) + if !checkpoint.FullSync || checkpoint.FullSyncBaseline["1"]["p2"] != "v:1" || + checkpoint.SpacePages["1"]["p2"] != "" { + t.Fatalf("failed page was not left pending in the full-sync checkpoint: %#v", checkpoint) + } + + resumeAPI := &streamAPI{pages: pages} + second := &captureHandler{} + next, err := newStreamConnector(resumeAPI).FetchFullStream( + context.Background(), streamConfig(), first.checkpoints[0], second, + ) + if err != nil { + t.Fatal(err) + } + if resumeAPI.bodyCalls != 2 { + t.Fatalf("retry fetched %d bodies, want failed p2 and uncheckpointed p3", resumeAPI.bodyCalls) + } + if len(second.items) != 2 || second.items[0].ExternalID != "p2" || second.items[1].ExternalID != "p3" { + t.Fatalf("retry items = %#v; failed page was skipped", second.items) + } + decoded := decodeCursor(next) + if decoded.FullSync || decoded.FullSyncBaseline != nil || len(decoded.SpacePages) != 1 || + decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "v:2" || + decoded.SpacePages["1"]["p3"] != "v:1" { + t.Fatalf("completed full-sync cursor = %#v", decoded) + } +} + func TestFetchStreamContinuesAfterPageBodyError(t *testing.T) { api := &streamAPI{ pages: []streamPage{ @@ -430,57 +718,536 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) { } } -func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) { - for _, full := range []bool{false, true} { - t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) { - api := &streamAPI{ - endlessNext: true, - pages: []streamPage{{id: "p1", title: "Page", version: 2}}, - probe: map[string]probeReply{ - "gone": {status: http.StatusNotFound}, - "moved": {status: http.StatusOK, spaceKey: "OPS"}, - "kept": {status: http.StatusOK, spaceKey: "ENG"}, - "denied": {status: http.StatusForbidden}, - }, - } - h := &captureHandler{} - old := streamCursor(map[string]string{ - "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1", - }) - connector := newStreamConnector(api) - fetch := connector.FetchStream - if full { - fetch = connector.FetchFullStream - } - got, err := fetch(context.Background(), streamConfig(), old, h) - if err != nil { - t.Fatalf("fetch: %v", err) - } - if api.listCalls != 1+maxEmptyServerPages { - t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages) - } - if api.probeCalls != 4 { - t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls) - } - deleted := map[string]bool{} - for _, item := range h.items { - if item.IsDeleted { - deleted[item.ExternalID] = true - } - } - if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] { - t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted) - } - pages := decodeCursor(got).SpacePages["1"] - want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"} - if len(pages) != len(want) { - t.Fatalf("cursor pages = %#v, want %#v", pages, want) - } - for id, version := range want { - if pages[id] != version { - t.Fatalf("cursor pages = %#v, want %#v", pages, want) - } - } - }) +func TestListResourcesLoadsConfluencePagesLazily(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Home", version: 1}}} + connector := newStreamConnector(api) + spaces, err := connector.ListResources(context.Background(), streamConfig(), "") + if err != nil || len(spaces) != 1 || !spaces[0].HasChildren { + t.Fatalf("spaces = %#v, %v", spaces, err) + } + pages, err := connector.ListResources(context.Background(), streamConfig(), "1") + if err != nil || len(pages) != 1 { + t.Fatalf("top pages = %#v, %v", pages, err) + } + if pages[0].ExternalID != "page:1:p1" || pages[0].ParentID != "1" || pages[0].Type != "page" || + !pages[0].HasChildren { + t.Fatalf("page resource = %#v", pages[0]) + } + children, err := connector.ListResources(context.Background(), streamConfig(), "page:1:p1") + if err != nil || len(children) != 0 { + t.Fatalf("children = %#v, %v", children, err) + } + if api.listCalls != 1 { + t.Fatalf("space expansion made %d whole-space list calls", api.listCalls) + } + if api.ancestorCalls != 0 { + t.Fatalf("page expansion made %d unnecessary ancestor calls", api.ancestorCalls) + } +} + +func TestPickerSpaceCacheIsPartitionedByCredentialFingerprint(t *testing.T) { + api := &streamAPI{} + connector := newStreamConnector(api) + client, cfg, err := connector.configured(streamConfig()) + if err != nil { + t.Fatal(err) + } + if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil { + t.Fatal(err) + } + if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil { + t.Fatal(err) + } + rotated := cfg + rotated.secret = "rotated-password" + if _, err := connector.pickerSpaces(context.Background(), client, rotated); err != nil { + t.Fatal(err) + } + if api.spaceCalls != 2 { + t.Fatalf("space list calls = %d; want 2 for distinct credentials", api.spaceCalls) + } +} + +func TestPickerSpaceCachePrunesExpiredCredentialEntries(t *testing.T) { + api := &streamAPI{} + connector := newStreamConnector(api) + client, cfg, err := connector.configured(streamConfig()) + if err != nil { + t.Fatal(err) + } + expired := cfg + expired.secret = "expired-password" + connector.spaceCache.entries = map[string]cachedSpaces{ + pickerSpaceCacheKey(expired): {expires: time.Now().Add(-time.Second)}, + } + if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil { + t.Fatal(err) + } + if len(connector.spaceCache.entries) != 1 { + t.Fatalf("cache entries = %#v; expired credential entry was retained", connector.spaceCache.entries) + } +} + +func TestFetchStreamSyncsSelectedPageSubtree(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}} + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 1 || h.items[0].SourceResourceID != "page:1:p1" || h.items[0].ExternalID != "p1" { + t.Fatalf("items = %#v", h.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:2" { + t.Fatalf("cursor = %#v", decoded) + } +} + +func TestFetchStreamReconcilesCancelledSpaceScope(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}} + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "p2" { + t.Fatalf("items = %#v", h.items) + } + decoded := decodeCursor(next) + if _, oldScopePresent := decoded.SpacePages["1"]; oldScopePresent || + decoded.SpacePages["page:1:p1"]["p1"] != "v:2" { + t.Fatalf("cursor = %#v", decoded) + } +} + +func TestFetchStreamPreservesMovedPageWhenBodyFails(t *testing.T) { + api := &streamAPI{ + pages: []streamPage{{id: "p1", title: "Selected", version: 2}}, + bodyStatus: map[string]int{"p1": http.StatusNotFound}, + } + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h) + if err != nil { + t.Fatal(err) + } + decoded := decodeCursor(next) + if decoded.SpacePages["page:1:p1"]["p1"] != "v:1" { + t.Fatalf("prior version was not preserved: %#v", decoded) + } + deletedP2 := false + for _, item := range h.items { + if item.IsDeleted && item.ExternalID == "p1" { + t.Fatalf("failed page was deleted: %#v", h.items) + } + deletedP2 = deletedP2 || (item.IsDeleted && item.ExternalID == "p2") + } + if !deletedP2 { + t.Fatalf("cancelled scope page was not deleted: %#v", h.items) + } +} + +func TestFetchStreamDoesNotDeleteWhenNewScopeCannotEnumerate(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}} + connector := newStreamConnector(api) + originalFactory := connector.newClient + connector.newClient = func(cfg config) (*client, error) { + c, err := originalFactory(cfg) + if err != nil { + return nil, err + } + originalTransport := c.http.Transport + c.http.Transport = roundTripper(func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/wiki/rest/api/content/p1" && req.URL.Query().Get("expand") == "version,space" { + return &http.Response{ + StatusCode: http.StatusNotFound, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("missing")), + }, nil + } + return originalTransport.RoundTrip(req) + }) + return c, nil + } + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + h := &captureHandler{} + _, err := connector.FetchStream( + context.Background(), + ds, + streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}), + h, + ) + if err == nil { + t.Fatal("scope enumeration failure unexpectedly succeeded") + } + for _, item := range h.items { + if item.IsDeleted { + t.Fatalf("incomplete scope emitted a tombstone: %#v", h.items) + } + } +} + +func TestBuildSyncPlanNormalizesOverlappingScopes(t *testing.T) { + api := &streamAPI{} + client := &client{ + cfg: config{baseURL: "https://confluence.test/wiki"}, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/rest/api/space": + return api.jsonResponse( + map[string]any{"results": []any{map[string]any{"id": 1, "key": "ENG", "name": "Engineering"}}}, + ) + case "/wiki/rest/api/content/parent": + return api.jsonResponse( + map[string]any{"id": "parent", "space": map[string]any{"key": "ENG"}, "ancestors": []any{}}, + ) + case "/wiki/rest/api/content/child": + return api.jsonResponse( + map[string]any{ + "id": "child", + "space": map[string]any{"key": "ENG"}, + "ancestors": []any{map[string]any{"id": "parent"}}, + }, + ) + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + } + connector := NewConnector() + plan, err := connector.buildSyncPlan(context.Background(), client, []string{"page:1:child", "page:1:parent"}) + if err != nil { + t.Fatal(err) + } + if len(plan.Roots) != 1 || plan.Roots[0].ResourceID != "page:1:parent" { + t.Fatalf("page overlap plan = %#v", plan) + } + plan, err = connector.buildSyncPlan(context.Background(), client, []string{"1", "page:1:child"}) + if err != nil { + t.Fatal(err) + } + if len(plan.Roots) != 1 || plan.Roots[0].Kind != syncWholeSpace || plan.Roots[0].ResourceID != "1" { + t.Fatalf("space overlap plan = %#v", plan) + } +} + +func TestFetchStreamCloudSyncsSelectedPageSubtree(t *testing.T) { + api := &streamAPI{cloud: true, pages: []streamPage{{id: "p1", title: "Cloud", version: 3}}} + ds := cloudStreamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 1 || h.items[0].ExternalID != "p1" || h.items[0].SourceResourceID != "page:1:p1" { + t.Fatalf("items = %#v", h.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:3" { + t.Fatalf("cursor = %#v", decoded) + } +} + +func TestListResourcesCloudStaysLazyWithContainerLimitation(t *testing.T) { + api := &streamAPI{} + var listDepths []string + conn := &Connector{newClient: func(cfg config) (*client, error) { + return &client{ + cfg: cfg, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/api/v2/spaces": + return api.jsonResponse( + map[string]any{ + "results": []any{map[string]any{"id": "1", "key": "ENG", "name": "Engineering"}}, + }, + ) + case "/wiki/api/v2/spaces/1/pages": + listDepths = append(listDepths, req.URL.Query().Get("depth")) + // depth=0 surfaces root pages only; the folder and the page nested + // inside it are unreachable through this endpoint (CONFCLOUD-84275). + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "top", "type": "page", "title": "Top"}, + map[string]any{"id": "folder", "type": "folder", "title": "Folder"}, + }}) + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + }, nil + }} + // Cloud spaces carry the limitation marker so the picker can explain why + // pages under top-level containers are not selectable. + spaces, err := conn.ListResources(context.Background(), cloudStreamConfig(), "") + if err != nil { + t.Fatal(err) + } + if len(spaces) != 1 || spaces[0].Metadata["hierarchy_limitation"] != "cloud_top_level_containers" { + t.Fatalf("cloud space metadata = %#v", spaces) + } + resources, err := conn.ListResources(context.Background(), cloudStreamConfig(), "1") + if err != nil { + t.Fatal(err) + } + if len(resources) != 1 || resources[0].ExternalID != "page:1:top" { + t.Fatalf("resources = %#v", resources) + } + // Expanding a space must stay a single shallow call, never a whole-space + // scan, even though top-level folders hide some pages. + if len(listDepths) != 1 || listDepths[0] != "0" { + t.Fatalf("space expansion depths = %#v; want a single depth=0 call", listDepths) + } +} + +func TestResolveResourceAncestorsSkipsUnresolvableSelections(t *testing.T) { + api := &streamAPI{} + connector := &Connector{newClient: func(cfg config) (*client, error) { + return &client{ + cfg: cfg, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/rest/api/space": + return api.jsonResponse(map[string]any{"results": []any{map[string]any{ + "id": 1, "key": "ENG", "name": "Engineering", + }}}) + case "/wiki/rest/api/content/alive": + return api.jsonResponse(map[string]any{ + "id": "alive", "space": map[string]any{"key": "ENG"}, + "ancestors": []any{map[string]any{"id": "parent"}}, + }) + case "/wiki/rest/api/content/gone": + return &http.Response{ + StatusCode: http.StatusNotFound, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("missing page")), + }, nil + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + }, nil + }} + ancestors, err := connector.ResolveResourceAncestors( + context.Background(), streamConfig(), + []string{"page:1:alive", "page:1:gone", "page:9:elsewhere", "not-a-resource"}, + ) + if err != nil { + t.Fatal(err) + } + want := []string{"1", "page:1:parent"} + if len(ancestors) != len(want) || ancestors[0] != want[0] || ancestors[1] != want[1] { + t.Fatalf("ancestors = %#v; want %#v", ancestors, want) + } +} + +func TestVisibleCloudPageChildrenTraversesContainers(t *testing.T) { + api := &streamAPI{} + client := &client{ + cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"}, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/api/v2/pages/a/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "b", "type": "page", "title": "B"}, + map[string]any{"id": "folder", "type": "folder", "title": "Folder"}, + }}) + case "/wiki/api/v2/folders/folder/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "c", "type": "page", "title": "C"}, + map[string]any{"id": "db", "type": "database", "title": "DB"}, + }}) + case "/wiki/api/v2/databases/db/direct-children": + return api.jsonResponse( + map[string]any{"results": []any{map[string]any{"id": "e", "type": "page", "title": "E"}}}, + ) + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + } + pages, err := client.visibleCloudPageChildren(context.Background(), "a") + if err != nil { + t.Fatal(err) + } + if len(pages) != 3 || pages[0].ID != "b" || pages[1].ID != "c" || pages[2].ID != "e" { + t.Fatalf("projected pages = %#v", pages) + } +} + +func TestCloudPageSubtreeWalksDeepMixedHierarchy(t *testing.T) { + api := &streamAPI{} + client := &client{ + cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"}, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + return nil, fmt.Errorf("unexpected method %s", req.Method) + } + switch req.URL.Path { + case "/wiki/api/v2/pages/root": + return api.jsonResponse(map[string]any{"id": "root", "spaceId": "1"}) + case "/wiki/api/v2/pages/p1": + return api.jsonResponse(map[string]any{ + "id": "p1", "spaceId": "1", "version": map[string]any{"number": 2}, + }) + case "/wiki/api/v2/pages/p2": + return api.jsonResponse(map[string]any{ + "id": "p2", "spaceId": "1", "version": map[string]any{"number": 3}, + }) + case "/wiki/api/v2/pages/p3": + return api.jsonResponse(map[string]any{ + "id": "p3", "spaceId": "1", "version": map[string]any{"number": 4}, + }) + case "/wiki/api/v2/pages/root/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "p1", "type": "page", "title": "First page"}, + map[string]any{"id": "folder", "type": "folder", "title": "Folder"}, + }}) + case "/wiki/api/v2/pages/p1/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "db", "type": "database", "title": "Database"}, + }}) + case "/wiki/api/v2/folders/folder/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "p2", "type": "page", "title": "Second page"}, + }}) + case "/wiki/api/v2/databases/db/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "p3", "type": "page", "title": "Deep page"}, + }}) + case "/wiki/api/v2/pages/p2/direct-children", "/wiki/api/v2/pages/p3/direct-children": + return api.jsonResponse(map[string]any{"results": []any{}}) + default: + return nil, fmt.Errorf("unexpected endpoint: %s", req.URL.String()) + } + })}, + } + + pages, err := client.pageSubtree(context.Background(), space{ID: "1"}, "root") + if err != nil { + t.Fatal(err) + } + want := []string{"root", "p1", "p2", "p3"} + if len(pages) != len(want) { + t.Fatalf("subtree pages = %#v; want IDs %v", pages, want) + } + for i, id := range want { + if pages[i].ID != id || pages[i].SpaceID != "1" { + t.Fatalf("subtree page[%d] = %#v; want ID %q in space 1", i, pages[i], id) + } + } +} + +func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) { + api := &streamAPI{ + endlessNext: true, + pages: []streamPage{{id: "p1", title: "Page", version: 2}}, + probe: map[string]probeReply{ + "gone": {status: http.StatusNotFound}, + "kept": {status: http.StatusOK, spaceKey: "ENG"}, + }, + } + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "v:1", "gone": "v:1", "kept": "v:1"}) + got, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if api.listCalls != 1+maxEmptyServerPages || api.probeCalls != 2 { + t.Fatalf("list/probe calls = %d/%d", api.listCalls, api.probeCalls) + } + if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "gone" { + t.Fatalf("items = %#v", h.items) + } + pages := decodeCursor(got).SpacePages["1"] + if pages["p1"] != "v:2" || pages["kept"] != "v:1" || len(pages) != 2 { + t.Fatalf("cursor pages = %#v", pages) + } +} + +func TestFetchStreamDoesNotRestoreCancelledRootForPageFoundByPartialProbe(t *testing.T) { + api := &streamAPI{ + endlessNext: true, + probe: map[string]probeReply{ + "kept": {status: http.StatusOK, spaceKey: "ENG"}, + }, + } + h := &captureHandler{} + old := (&cursor{SpacePages: map[string]map[string]string{ + "page:1:old-root": {"kept": "v:1"}, + }}).syncCursor() + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 0 { + t.Fatalf("confirmed-present page emitted items: %#v", h.items) + } + decoded := decodeCursor(next) + if len(decoded.SpacePages) != 1 || decoded.SpacePages["1"]["kept"] != "v:1" { + t.Fatalf("reconciled cursor = %#v; expected only the current root", decoded.SpacePages) + } + if _, oldRootPresent := decoded.SpacePages["page:1:old-root"]; oldRootPresent { + t.Fatalf("cancelled root was restored to cursor: %#v", decoded.SpacePages) + } +} + +func TestFetchStreamDoesNotTombstoneWhenPartialListingProbeFails(t *testing.T) { + api := &streamAPI{ + endlessNext: true, + probe: map[string]probeReply{ + "gone": {status: http.StatusNotFound}, + "unknown": {status: http.StatusForbidden}, + }, + } + h := &captureHandler{} + old := streamCursor(map[string]string{"gone": "v:1", "unknown": "v:1"}) + _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err == nil { + t.Fatal("unverified partial-listing candidate unexpectedly completed reconciliation") + } + for _, item := range h.items { + if item.IsDeleted { + t.Fatalf("probe failure allowed a tombstone: %#v", h.items) + } + } + if len(h.checkpoints) != 0 { + t.Fatalf("probe failure checkpointed reconciliation: %#v", h.checkpoints) + } + if api.probeCalls == 0 { + t.Fatal("empty incomplete listing was rejected before checking prior pages") + } +} + +func TestFetchStreamDefersWhenPartialListingProbeBudgetIsExhausted(t *testing.T) { + prior := make(map[string]string, maxDeletionProbes+1) + probes := make(map[string]probeReply, maxDeletionProbes+1) + for i := 0; i <= maxDeletionProbes; i++ { + id := fmt.Sprintf("p%03d", i) + prior[id] = "v:1" + probes[id] = probeReply{status: http.StatusNotFound} + } + api := &streamAPI{endlessNext: true, probe: probes} + h := &captureHandler{} + _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h) + if err == nil { + t.Fatal("partial listing with unverified candidates unexpectedly completed") + } + if api.probeCalls != maxDeletionProbes { + t.Fatalf("existence probes = %d; want capped at %d", api.probeCalls, maxDeletionProbes) + } + if len(h.items) != 0 || len(h.checkpoints) != 0 { + t.Fatalf( + "probe budget exhaustion emitted or checkpointed reconciliation: items=%#v checkpoints=%#v", + h.items, + h.checkpoints, + ) } } diff --git a/internal/datasource/connector/confluence/resource.go b/internal/datasource/connector/confluence/resource.go new file mode 100644 index 00000000..a34a1619 --- /dev/null +++ b/internal/datasource/connector/confluence/resource.go @@ -0,0 +1,43 @@ +package confluence + +import ( + "fmt" + "strings" +) + +type resourceKind uint8 + +const ( + resourceSpace resourceKind = iota + resourcePage +) + +// resourceRef is the canonical, persisted representation of a selection. +// Space IDs deliberately remain unprefixed for backwards compatibility. +type resourceRef struct { + Kind resourceKind + SpaceID string + PageID string +} + +func makePageResourceID(spaceID, pageID string) string { + return "page:" + spaceID + ":" + pageID +} + +func parseResourceID(id string) (resourceRef, error) { + id = strings.TrimSpace(id) + if id == "" { + return resourceRef{}, fmt.Errorf("invalid empty Confluence resource ID") + } + if !strings.HasPrefix(id, "page:") { + if strings.Contains(id, ":") { + return resourceRef{}, fmt.Errorf("invalid Confluence space resource ID %q", id) + } + return resourceRef{Kind: resourceSpace, SpaceID: id}, nil + } + parts := strings.Split(id, ":") + if len(parts) != 3 || parts[0] != "page" || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" { + return resourceRef{}, fmt.Errorf("invalid Confluence page resource ID %q", id) + } + return resourceRef{Kind: resourcePage, SpaceID: parts[1], PageID: parts[2]}, nil +} diff --git a/internal/datasource/connector/confluence/resource_test.go b/internal/datasource/connector/confluence/resource_test.go new file mode 100644 index 00000000..b03e92f2 --- /dev/null +++ b/internal/datasource/connector/confluence/resource_test.go @@ -0,0 +1,23 @@ +package confluence + +import "testing" + +func TestParseResourceID(t *testing.T) { + space, err := parseResourceID("86802433") + if err != nil || space.Kind != resourceSpace || space.SpaceID != "86802433" { + t.Fatalf("space = %#v, %v", space, err) + } + page, err := parseResourceID("page:86802433:123456789") + if err != nil || page.Kind != resourcePage || page.SpaceID != "86802433" || page.PageID != "123456789" { + t.Fatalf("page = %#v, %v", page, err) + } + invalid := []string{ + "", "page:", "page:100", "page::200", "page:100:", "page:100:200:300", + "node:86802433:folder-1", "space:100", + } + for _, id := range invalid { + if _, err := parseResourceID(id); err == nil { + t.Errorf("parseResourceID(%q) succeeded", id) + } + } +} diff --git a/internal/datasource/connector/confluence/sync_plan.go b/internal/datasource/connector/confluence/sync_plan.go new file mode 100644 index 00000000..c61484d9 --- /dev/null +++ b/internal/datasource/connector/confluence/sync_plan.go @@ -0,0 +1,112 @@ +package confluence + +import ( + "context" + "fmt" + "sort" +) + +type syncRootKind uint8 + +const ( + syncWholeSpace syncRootKind = iota + syncPageSubtree +) + +type syncRoot struct { + ResourceID string + SpaceID string + Space space + PageID string + Kind syncRootKind +} + +type syncPlan struct{ Roots []syncRoot } + +// buildSyncPlan validates selections before any destructive reconciliation and +// removes scopes which are already covered by a selected ancestor. +func (c *Connector) buildSyncPlan(ctx context.Context, client *client, resourceIDs []string) (*syncPlan, error) { + spaces, err := client.spaces(ctx) + if err != nil { + return nil, fmt.Errorf("list Confluence spaces: %w", err) + } + byID := make(map[string]space, len(spaces)) + for _, s := range spaces { + byID[s.ID] = s + } + + plan := &syncPlan{} + wholeSpaces := make(map[string]bool) + pageRoots := make(map[string]syncRoot) + ancestorPaths := make(map[string][]page) + for _, resourceID := range resourceIDs { + ref, err := parseResourceID(resourceID) + if err != nil { + return nil, err + } + s, ok := byID[ref.SpaceID] + if !ok { + return nil, fmt.Errorf( + "selected Confluence space %s is unavailable; refusing destructive reconciliation", + ref.SpaceID, + ) + } + if ref.Kind == resourceSpace { + wholeSpaces[ref.SpaceID] = true + continue + } + // pageAncestors fetches the page itself on Server/DC and Cloud; that is + // also the authoritative check that a client did not forge its space ID. + ancestors, err := client.pageAncestors(ctx, s, ref.PageID) + if err != nil { + return nil, fmt.Errorf("validate selected Confluence page %s: %w", ref.PageID, err) + } + pageRoots[resourceID] = syncRoot{ + ResourceID: resourceID, + SpaceID: ref.SpaceID, + Space: s, + PageID: ref.PageID, + Kind: syncPageSubtree, + } + ancestorPaths[resourceID] = ancestors + for _, ancestor := range ancestors { + if ancestor.ID == ref.PageID { + return nil, fmt.Errorf("invalid cyclic Confluence page ancestry for %s", ref.PageID) + } + } + } + spaceIDs := make([]string, 0, len(wholeSpaces)) + for id := range wholeSpaces { + spaceIDs = append(spaceIDs, id) + } + sort.Strings(spaceIDs) + for _, id := range spaceIDs { + plan.Roots = append(plan.Roots, syncRoot{ResourceID: id, SpaceID: id, Space: byID[id], Kind: syncWholeSpace}) + } + pageIDs := make([]string, 0, len(pageRoots)) + for id := range pageRoots { + pageIDs = append(pageIDs, id) + } + sort.Strings(pageIDs) + for _, id := range pageIDs { + root := pageRoots[id] + if wholeSpaces[root.SpaceID] { + continue + } + ancestors := ancestorPaths[id] + covered := false + for _, ancestor := range ancestors { + if ancestor.Kind != "" && ancestor.Kind != "page" { + continue + } + if _, ok := pageRoots[makePageResourceID(root.SpaceID, ancestor.ID)]; ok { + covered = true + break + } + } + if !covered { + plan.Roots = append(plan.Roots, pageRoots[id]) + } + } + return plan, nil +} diff --git a/internal/datasource/connector/confluence/types.go b/internal/datasource/connector/confluence/types.go index 7884f349..49828808 100644 --- a/internal/datasource/connector/confluence/types.go +++ b/internal/datasource/connector/confluence/types.go @@ -95,10 +95,11 @@ func isAtlassianCloudHost(host string) bool { } type space struct { - ID string `json:"id"` - Key string `json:"key"` - Name string `json:"name"` - Links struct { + ID string `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + HomepageID string `json:"homepageId"` + Links struct { WebUI string `json:"webui"` } `json:"_links"` } @@ -125,10 +126,13 @@ type serverSpaceList struct { } type page struct { - ID string `json:"id"` - Title string `json:"title"` - Status string `json:"status"` - Space struct { + ID string `json:"id"` + Title string `json:"title"` + Status string `json:"status"` + Kind string `json:"-"` + SpaceID string `json:"spaceId"` + Ancestors []page `json:"ancestors"` + Space struct { Key string `json:"key"` Name string `json:"name"` } `json:"space"` @@ -189,6 +193,27 @@ type cloudPage struct { } `json:"_links"` } +// cloudHierarchyItem is shared by Cloud direct-children, descendants, and +// ancestors. Non-page items are intentionally retained by the API model so we +// can filter them explicitly at the navigation/import boundary. +type cloudHierarchyItem struct { + ID string `json:"id"` + Type string `json:"type"` + Title string `json:"title"` + SpaceID string `json:"spaceId"` + Status string `json:"status"` + Links struct { + WebUI string `json:"webui"` + } `json:"_links"` +} + +type cloudHierarchyList struct { + Results []cloudHierarchyItem `json:"results"` + Links struct { + Next string `json:"next"` + } `json:"_links"` +} + type cloudPageList struct { Results []cloudPage `json:"results"` Links struct { @@ -278,14 +303,37 @@ func prepareSyncCursors(old *types.SyncCursor, forceFull bool) (baseline, next c return baseline, next } -func pageVersion(p page) string { +// removePageOwnership drops a page from every root ownership in the cursor. +// It runs after each tombstone emit so the checkpoint mirrors the deletion; +// otherwise the page would linger as a ghost, be re-tombstoned on every later +// run, and eventually trip the mass-deletion guard. During a full sync the +// deletion baseline lives in FullSyncBaseline, not SpacePages (which is being +// rebuilt from scratch), so both maps must record the completed tombstone for +// a retry to resume with the remaining candidates only. +func removePageOwnership(c *cursor, pageID string) { + removeFromPageMap(c.SpacePages, pageID) + if c.FullSync { + removeFromPageMap(c.FullSyncBaseline, pageID) + } +} + +func removeFromPageMap(m map[string]map[string]string, pageID string) { + for _, pages := range m { + delete(pages, pageID) + } +} + +func pageVersion(p page) (string, bool) { if p.Version.Number > 0 { - return fmt.Sprintf("v:%d", p.Version.Number) + return fmt.Sprintf("v:%d", p.Version.Number), true } if p.Version.When != "" { - return "t:" + p.Version.When + return "t:" + p.Version.When, true } - return "t:" + p.Version.CreatedAt + if p.Version.CreatedAt != "" { + return "t:" + p.Version.CreatedAt, true + } + return "", false } func pageUpdatedAt(p page) time.Time { diff --git a/internal/datasource/connector/confluence/types_test.go b/internal/datasource/connector/confluence/types_test.go index 6a2247f0..bf62f7f3 100644 --- a/internal/datasource/connector/confluence/types_test.go +++ b/internal/datasource/connector/confluence/types_test.go @@ -75,6 +75,34 @@ func TestPageFileNameIncludesID(t *testing.T) { } } +func TestPageVersionUnknown(t *testing.T) { + withNumber := page{} + withNumber.Version.Number = 7 + withWhen := page{} + withWhen.Version.When = "2026-01-02T03:04:05Z" + withCreatedAt := page{} + withCreatedAt.Version.CreatedAt = "2026-01-02T03:04:05Z" + + for _, tc := range []struct { + name string + in page + want string + wantKnown bool + }{ + {"number", withNumber, "v:7", true}, + {"when", withWhen, "t:2026-01-02T03:04:05Z", true}, + {"createdAt", withCreatedAt, "t:2026-01-02T03:04:05Z", true}, + {"unknown", page{}, "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, known := pageVersion(tc.in) + if got != tc.want || known != tc.wantKnown { + t.Fatalf("pageVersion() = (%q, %v); want (%q, %v)", got, known, tc.want, tc.wantKnown) + } + }) + } +} + func TestPrepareSyncCursorsPreservesFullSyncBaseline(t *testing.T) { old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) baseline, next := prepareSyncCursors(old, true)