diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts index 1ac7ac6d..19096307 100644 --- a/frontend/src/api/datasource/index.ts +++ b/frontend/src/api/datasource/index.ts @@ -82,6 +82,8 @@ export interface Resource { url: string parent_id?: string has_children?: boolean + /** Connector-specific hints, e.g. Confluence Cloud's top-level-container limitation. */ + metadata?: Record } // --- API calls --- diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 1c7e6dcf..681150b9 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -6979,6 +6979,9 @@ export default { authHeaders: 'Custom headers (optional)', authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud cannot list pages stored directly under top-level folders in this space; selecting the whole space still syncs them.' + }, comingSoon: 'Coming soon', docHint: 'Get credentials at:', openDoc: 'Open documentation', diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts index ac272ac6..308b3c0f 100644 --- a/frontend/src/i18n/locales/ja-JP.ts +++ b/frontend/src/i18n/locales/ja-JP.ts @@ -6979,6 +6979,9 @@ export default { authHeaders: 'カスタムヘッダー(任意)', authHeadersHint: '非公開フィード用です。1行に1つ「名前: 値」の形式で入力します(例: Authorization: Bearer xxxx)' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud では、スペース直下のフォルダーなどコンテナ配下のページをここに一覧できません。スペース全体を選択すれば同期されます。' + }, comingSoon: '近日対応予定', docHint: '認証情報の取得先:', openDoc: 'ドキュメントを開く', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index ead7f713..9bba3b96 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1125,6 +1125,9 @@ export default { authHeaders: '사용자 지정 헤더 (선택)', authHeadersHint: '비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud는 아직 스페이스 최상위 폴더 등 컨테이너 아래의 페이지를 여기에 나열할 수 없습니다. 스페이스 전체를 선택하면 함께 동기화됩니다.' + }, connectorDesc: { feishu: '페이슈 위키에서 문서, 스프레드시트, 파일 동기화', lark: 'Lark 위키에서 문서, 스프레드시트, 파일 동기화', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 4f075e0b..d5b8fb16 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1125,6 +1125,9 @@ export default { authHeaders: 'Пользовательские заголовки (необязательно)', authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud пока не может показать здесь страницы из папок верхнего уровня пространства; при выборе всего пространства они будут синхронизированы.' + }, connectorDesc: { feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki', lark: 'Синхронизация документов, таблиц и файлов из Lark Wiki', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 073f8649..5497981a 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1127,6 +1127,9 @@ export default { authHeaders: '自定义请求头(可选)', authHeadersHint: '用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx' }, + confluence: { + cloudFolderLimitation: 'Confluence Cloud 暂不支持在此列出空间顶层文件夹等容器下的页面;选择整个空间仍会同步它们。' + }, connectorDesc: { feishu: '同步飞书知识库中的文档、表格、文件', lark: '同步 Lark 知识库中的文档、表格、文件(飞书国际版)', diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts index 43c46c76..f9f2d126 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts @@ -157,3 +157,25 @@ test('new GitLab data sources continue to test credentials without persistence', assert.deepEqual(f.calls.map(call => call.method), ['validateCredentials']) } finally { f.close() } }) + +test('Cloud hierarchy limitation stays visible after an empty space expansion', async () => { + const f = await fixture() + try { + f.vm.resources = [{ + external_id: 'space-1', + name: 'Cloud space', + type: 'space', + has_children: true, + metadata: { hierarchy_limitation: 'cloud_top_level_containers' }, + }] + f.vm.expandedResourceIds = new Set(['space-1']) + await nextTick() + assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true) + + f.vm.expandedResourceIds = new Set() + f.vm.loadedChildrenIds = new Set(['space-1']) + f.vm.resources = [{ ...f.vm.resources[0], has_children: false }] + await nextTick() + assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true) + } finally { f.close() } +}) diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue index 44b16aa4..cb60d2bb 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue @@ -449,6 +449,16 @@ async function ensureChildrenLoaded(id: string) { if (!existing.has(c.external_id)) merged.push(c) } resources.value = merged + } else { + // Connectors may conservatively advertise HasChildren to avoid an N+1 + // probe. Once lazy loading proves this is a leaf, collapse it and retain + // that fact so another expand cannot trigger another empty request. + resources.value = resources.value.map(r => r.external_id === id + ? { ...r, has_children: false } + : r) + const expanded = new Set(expandedResourceIds.value) + expanded.delete(id) + expandedResourceIds.value = expanded } loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id) } catch (e: any) { @@ -466,13 +476,21 @@ async function ensureChildrenLoaded(id: string) { const visibleTree = computed(() => { const roots = resources.value.filter(r => !r.parent_id) - const result: { resource: Resource; depth: number }[] = [] + const result: { resource: Resource; depth: number; noticeAfter?: boolean }[] = [] function walk(items: Resource[], depth: number) { for (const r of items) { result.push({ resource: r, depth }) if (r.has_children && expandedResourceIds.value.has(r.external_id)) { walk(childrenMap.value.get(r.external_id) || [], depth + 1) } + // Keep this visible after an empty root listing is recognized as a leaf. + if ( + r.metadata?.hierarchy_limitation === 'cloud_top_level_containers' && + (expandedResourceIds.value.has(r.external_id) || + (!r.has_children && loadedChildrenIds.value.has(r.external_id))) + ) { + result.push({ resource: r, depth: depth + 1, noticeAfter: true }) + } } } walk(roots, 0) @@ -1720,9 +1738,12 @@ const drawerConfirmText = computed(() => {
+
@@ -2576,6 +2605,19 @@ const drawerConfirmText = computed(() => { background: color-mix(in srgb, var(--td-text-color-placeholder) 8%, transparent); } +.resource-picker__notice { + --depth: 0; + margin: 0 0 2px; + padding: 4px 8px 4px calc(8px + var(--depth) * 14px); + font-size: var(--app-text-xs); + line-height: 1.5; + color: var(--td-text-color-placeholder); +} + +.resource-picker__notice:last-child { + margin-bottom: 0; +} + /* --- Step 2: empty state --- */ .ds-resource-empty { text-align: center; diff --git a/internal/datasource/connector/confluence/asset_resolver.go b/internal/datasource/connector/confluence/asset_resolver.go new file mode 100644 index 00000000..f8b06f60 --- /dev/null +++ b/internal/datasource/connector/confluence/asset_resolver.go @@ -0,0 +1,301 @@ +package confluence + +import ( + "context" + "encoding/base64" + "errors" + "html" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + + "github.com/Tencent/WeKnora/internal/logger" +) + +const ( + // maxPageImages is the total data URI image budget one page may carry into the + // downstream docparser, which enforces a single per-page limit (maxRemoteImages) + // across every data:image URI regardless of origin. The budget is shared: it + // covers data URI images already present in the page plus the ones the resolver + // generates, and a repeated image consumes it per occurrence, not per unique URL. + // Downloads stay deduplicated by URL, and only occurrences within the remaining + // budget are collected, so the connector never emits more data URIs than the + // pipeline will keep. + maxPageImages = 30 + // maxPageInlineBytes caps the total size of the data URIs one page may carry, + // both in the download cache and in the rewritten HTML. Without it an extreme + // page (maxPageImages occurrences of maxImageBytes images) would inline hundreds + // of MB of base64, copied again by every downstream conversion. Images past the + // budget keep their original src, like any other image the resolver skips. + maxPageInlineBytes = 50 << 20 + // imageDownloadConcurrency bounds in-flight image downloads per page so an + // image-heavy page cannot exhaust HTTP connections or spike memory. + imageDownloadConcurrency = 4 +) + +// errImagePageBudget marks an image dropped because the page's inline budget +// (maxPageInlineBytes) was already used up. +var errImagePageBudget = errors.New("confluence page image budget exhausted") + +// imgTagRe matches a full start tag, tolerating '>' inside quoted +// attribute values. The \b keeps it from matching . +var imgTagRe = regexp.MustCompile(`(?i)"']|"[^"]*"|'[^']*')*>`) + +// srcAttrRe captures the src attribute value. The leading \s and the value group +// keep it from matching data-src/srcset, and support double-quoted, +// single-quoted, or bare values. Group 1 is the "src=" prefix, group 2 the value. +var srcAttrRe = regexp.MustCompile(`(?i)(\ssrc\s*=\s*)("[^"]*"|'[^']*'|[^\s"'>]+)`) + +type imgAction uint8 + +const ( + imgKeep imgAction = iota // empty, data:, external, or non-http scheme: leave untouched + imgDownload // same-origin private candidate +) + +// assetResolver rewrites same-origin private Confluence images in a page's +// body.view HTML into inline data URIs. Persistence and resource:// rewriting +// are left entirely to the downstream docparser image pipeline; the resolver +// only performs the one step that pipeline cannot do on its own: an +// authenticated fetch of Confluence-private bytes. +type assetResolver struct { + client *client +} + +func newAssetResolver(c *client) *assetResolver { return &assetResolver{client: c} } + +// Resolve returns content with same-origin private occurrences replaced +// by data URIs, within the per-page budget that Resolve shares with any data URI +// images already in the page. It is best-effort: an image that is skipped, falls +// past the remaining budget, or fails to download keeps its original src, and image +// problems never fail the page. Downloads are deduplicated by URL, but the budget +// counts written data URIs (occurrences) to match the downstream pipeline, which +// caps every data:image URI per page regardless of where it came from. +func (r *assetResolver) Resolve(ctx context.Context, content string) string { + if r == nil || r.client == nil { + return content + } + // Data URI images already in the page consume the same downstream budget as the + // ones we generate, so reserve room for them before inlining any private image. + available := maxPageImages - countDataURIImages(content) + if available <= 0 { + return content + } + candidates := r.collect(content, available) + if len(candidates) == 0 { + return content + } + cache := r.downloadAll(ctx, candidates) + if len(cache) == 0 { + return content + } + inlined, inlinedBytes := 0, 0 + budgetSpent := false + return imgTagRe.ReplaceAllStringFunc(content, func(tag string) string { + if inlined >= available || budgetSpent { + return tag + } + src, ok := extractImageSrc(tag) + if !ok { + return tag + } + action, abs := r.classify(src) + if action != imgDownload { + return tag + } + dataURI := cache[abs] + if dataURI == "" { + return tag + } + if inlinedBytes+len(dataURI) > maxPageInlineBytes { + budgetSpent = true + return tag + } + inlined++ + inlinedBytes += len(dataURI) + return replaceImageSrc(tag, dataURI) + }) +} + +// collect returns the unique same-origin download candidates whose first +// occurrence falls within the remaining inline budget, in document order. Because +// Resolve writes at most budget data URIs, a URL first seen after that many +// occurrences can never be inlined, so it is not downloaded. Duplicates collapse to +// one entry, so a repeated image is fetched once yet may be inlined at every +// occurrence inside the budget. +func (r *assetResolver) collect(content string, budget int) []string { + tags := imgTagRe.FindAllString(content, -1) + order := make([]string, 0, len(tags)) + seen := make(map[string]struct{}, len(tags)) + occurrences := 0 + for _, tag := range tags { + src, ok := extractImageSrc(tag) + if !ok { + continue + } + action, abs := r.classify(src) + if action != imgDownload { + continue + } + if occurrences >= budget { + break + } + occurrences++ + if _, dup := seen[abs]; dup { + continue + } + seen[abs] = struct{}{} + order = append(order, abs) + } + return order +} + +// countDataURIImages returns how many tags in content already carry a +// data:image URI. The downstream docparser counts every data:image occurrence +// against the same per-page limit whether it came from the page or from us, so +// these must be reserved before the resolver inlines any private Confluence image. +func countDataURIImages(content string) int { + count := 0 + for _, tag := range imgTagRe.FindAllString(content, -1) { + src, ok := extractImageSrc(tag) + if !ok { + continue + } + if strings.HasPrefix(strings.ToLower(src), "data:image/") { + count++ + } + } + return count +} + +// downloadAll fetches each candidate with bounded concurrency and returns a +// URL-keyed cache of the successful ones as data URIs, holding at most +// maxPageInlineBytes of them. Failures and images past the budget are logged +// (redacted) and omitted, so the rewrite pass leaves their original src in place. +func (r *assetResolver) downloadAll(ctx context.Context, urls []string) map[string]string { + cache := make(map[string]string, len(urls)) + cachedBytes := 0 + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, imageDownloadConcurrency) + for _, u := range urls { + if ctx.Err() != nil { + break + } + wg.Add(1) + sem <- struct{}{} + go func(abs string) { + defer wg.Done() + defer func() { <-sem }() + data, mime, err := r.client.downloadImage(ctx, abs) + if err != nil { + logImageFailure(ctx, abs, err) + return + } + uri := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data) + mu.Lock() + fits := cachedBytes+len(uri) <= maxPageInlineBytes + if fits { + cache[abs] = uri + cachedBytes += len(uri) + } + mu.Unlock() + if !fits { + logImageFailure(ctx, abs, errImagePageBudget) + } + }(u) + } + wg.Wait() + return cache +} + +// classify decides whether src is a same-origin private candidate to download or +// should be left untouched, returning the normalized absolute URL for downloads. +func (r *assetResolver) classify(src string) (imgAction, string) { + s := strings.TrimSpace(src) + if s == "" || strings.HasPrefix(strings.ToLower(s), "data:") { + return imgKeep, "" + } + u, err := url.Parse(s) + if err != nil { + return imgKeep, "" + } + if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" { + // file:, javascript:, and any other scheme are never fetched. + return imgKeep, "" + } + if u.Host != "" { + if u.Scheme == "" { + // Protocol-relative //host/path: only same-origin is eligible. + base, berr := url.Parse(r.client.cfg.baseURL) + if berr != nil || !strings.EqualFold(base.Host, u.Host) { + return imgKeep, "" + } + } else if !r.client.sameOrigin(u) { + // Absolute external http(s): keep, never send Confluence credentials. + return imgKeep, "" + } + } + abs, err := r.client.resolveEndpoint(s) + if err != nil { + return imgKeep, "" + } + return imgDownload, abs +} + +// logImageFailure records a redacted warning: host and path only, never the +// query (which may carry signed tokens), the Authorization header, or the body. +func logImageFailure(ctx context.Context, abs string, err error) { + host, path := abs, "" + if u, perr := url.Parse(abs); perr == nil { + host, path = u.Host, u.Path + } + reason := "download_failed" + switch { + case errors.Is(err, errImageTooLarge): + reason = "oversize" + case errors.Is(err, errImageNonImageContent): + reason = "non_image_content_type" + case errors.Is(err, errImageRedirectOffOrigin): + reason = "cross_origin_redirect" + case errors.Is(err, errImagePageBudget): + reason = "page_budget" + } + var api *apiError + if errors.As(err, &api) { + reason = "status_" + strconv.Itoa(api.status) + } + logger.Warnf(ctx, "[Confluence] failed to resolve image host=%s path=%s reason=%s", host, path, reason) +} + +// extractImageSrc returns the HTML-unescaped src value of an tag. +func extractImageSrc(tag string) (string, bool) { + m := srcAttrRe.FindStringSubmatch(tag) + if m == nil { + return "", false + } + value := strings.TrimSpace(m[2]) + if len(value) >= 2 { + if q := value[0]; (q == '"' || q == '\'') && value[len(value)-1] == q { + value = value[1 : len(value)-1] + } + } + value = strings.TrimSpace(html.UnescapeString(value)) + if value == "" { + return "", false + } + return value, true +} + +// replaceImageSrc swaps only the src attribute value for a double-quoted data +// URI, preserving the rest of the tag byte-for-byte. +func replaceImageSrc(tag, dataURI string) string { + loc := srcAttrRe.FindStringSubmatchIndex(tag) + if loc == nil { + return tag + } + start, end := loc[4], loc[5] + return tag[:start] + `"` + dataURI + `"` + tag[end:] +} diff --git a/internal/datasource/connector/confluence/asset_resolver_test.go b/internal/datasource/connector/confluence/asset_resolver_test.go new file mode 100644 index 00000000..98b12483 --- /dev/null +++ b/internal/datasource/connector/confluence/asset_resolver_test.go @@ -0,0 +1,612 @@ +package confluence + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" +) + +const testBaseURL = "https://confluence.test/wiki" + +func newImageTestClient(rt roundTripper) *client { + return &client{ + cfg: config{baseURL: testBaseURL, username: "reader", secret: "secret"}, + http: &http.Client{Transport: rt}, + } +} + +func imageResp(status int, contentType string, body []byte) *http.Response { + header := make(http.Header) + if contentType != "" { + header.Set("Content-Type", contentType) + } + return &http.Response{ + StatusCode: status, + Header: header, + Body: io.NopCloser(bytes.NewReader(body)), + } +} + +func redirectResp(location string) *http.Response { + header := make(http.Header) + header.Set("Location", location) + return &http.Response{ + StatusCode: http.StatusFound, + Header: header, + Body: io.NopCloser(bytes.NewReader(nil)), + } +} + +// reqLog records outbound requests. Image downloads run concurrently, so every +// access is mutex-guarded. +type reqLog struct { + mu sync.Mutex + reqs []reqRecord +} + +type reqRecord struct { + host string + path string + hasAuth bool +} + +func newReqLog() *reqLog { return &reqLog{} } + +func (l *reqLog) add(req *http.Request) { + l.mu.Lock() + defer l.mu.Unlock() + l.reqs = append(l.reqs, reqRecord{ + host: req.URL.Host, + path: req.URL.Path, + hasAuth: req.Header.Get("Authorization") != "", + }) +} + +func (l *reqLog) count() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.reqs) +} + +func (l *reqLog) all() []reqRecord { + l.mu.Lock() + defer l.mu.Unlock() + return append([]reqRecord(nil), l.reqs...) +} + +func (l *reqLog) hasHost(host string) bool { + for _, r := range l.all() { + if r.host == host { + return true + } + } + return false +} + +func (l *reqLog) hasPath(path string) bool { + for _, r := range l.all() { + if r.path == path { + return true + } + } + return false +} + +func (l *reqLog) anyWithoutAuth() bool { + for _, r := range l.all() { + if !r.hasAuth { + return true + } + } + return false +} + +var testPNG = []byte("private-png-bytes") + +func testDataURI(body []byte) string { + return "data:image/png;base64," + base64.StdEncoding.EncodeToString(body) +} + +func TestAssetResolverKeepsPublicURL(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `

hi

` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("public URL was modified:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("public image was downloaded: %v", log.all()) + } +} + +func TestAssetResolverKeepsBase64(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("base64 image was modified:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("base64 image triggered a download: %v", log.all()) + } +} + +func TestAssetResolverKeepsNonHTTPScheme(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("non-http scheme was modified:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("non-http scheme triggered a download: %v", log.all()) + } +} + +func TestAssetResolverInlinesRelativeConfluenceImage(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + want := testDataURI(testPNG) + if !strings.Contains(out, `src="`+want+`"`) { + t.Fatalf("relative image not inlined as data URI:\n%s", out) + } + if strings.Contains(out, "/wiki/download/attachments") { + t.Fatalf("original private URL survived:\n%s", out) + } + reqs := log.all() + if len(reqs) != 1 { + t.Fatalf("expected 1 download, got %d: %v", len(reqs), reqs) + } + if reqs[0].host != "confluence.test" || !reqs[0].hasAuth { + t.Fatalf("download missing credentials or wrong host: %+v", reqs[0]) + } +} + +func TestAssetResolverInlinesSameOriginPrivateImage(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `x` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if !strings.Contains(out, `src="`+testDataURI(testPNG)+`"`) { + t.Fatalf("same-origin image not inlined:\n%s", out) + } + // The rest of the tag must be preserved. + if !strings.Contains(out, `alt="x"`) { + t.Fatalf("sibling attribute lost:\n%s", out) + } + if log.count() != 1 { + t.Fatalf("expected 1 download, got %d", log.count()) + } +} + +func TestAssetResolverDoesNotSendCredentialsCrossOrigin(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + + `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if log.hasHost("cdn.example.com") { + t.Fatalf("external host received a request: %v", log.all()) + } + if !strings.Contains(out, "https://cdn.example.com/a.png") { + t.Fatalf("external URL was modified:\n%s", out) + } + if !strings.Contains(out, testDataURI(testPNG)) { + t.Fatalf("same-origin image was not inlined:\n%s", out) + } + if log.count() != 1 || log.anyWithoutAuth() { + t.Fatalf("expected exactly one authenticated same-origin download: %v", log.all()) + } +} + +func TestAssetResolverRejectsCrossOriginRedirect(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + if req.URL.Host == "confluence.test" { + return redirectResp("https://evil.example.com/a.png"), nil + } + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("cross-origin redirect was followed and inlined:\n%s", out) + } + if log.hasHost("evil.example.com") { + t.Fatalf("credentials could have leaked to evil host: %v", log.all()) + } + if log.count() != 1 { + t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all()) + } +} + +func TestAssetResolverRejectsSameOriginRedirectOutsideContextPath(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + // The base URL is https://confluence.test/wiki. A redirect to another app on + // the same host stays same-origin but escapes the /wiki context path, and Go + // keeps sending Basic Auth on a same-host hop. It must be refused before the + // credentialed request is ever sent, not merely discarded afterward. + if strings.HasPrefix(req.URL.Path, "/wiki/") { + return redirectResp("https://confluence.test/other-app/leak.png"), nil + } + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("same-host redirect outside the context path was followed and inlined:\n%s", out) + } + if log.hasPath("/other-app/leak.png") { + t.Fatalf("credentials followed a redirect outside the context path: %v", log.all()) + } + if log.count() != 1 { + t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all()) + } +} + +func TestAssetResolverRejectsOversizedImage(t *testing.T) { + log := newReqLog() + big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1) + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", big), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("oversized image was inlined:\n%s", out) + } + if strings.Contains(out, "data:image") { + t.Fatalf("oversized image produced a data URI") + } +} + +func TestAssetResolverRejectsNonImageResponse(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "text/html; charset=utf-8", []byte("login required")), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if out != in { + t.Fatalf("login HTML was inlined as an image:\n%s", out) + } +} + +func TestAssetResolverKeepsOriginalURLOnDownloadFailure(t *testing.T) { + for _, status := range []int{http.StatusForbidden, http.StatusNotFound, http.StatusInternalServerError} { + t.Run(http.StatusText(status), func(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(status, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("status %d modified the document:\n%s", status, out) + } + }) + } +} + +func TestAssetResolverDeduplicatesSamePageImages(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + src := `` + in := strings.Repeat(src, 5) + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if got := strings.Count(out, "data:image/png;base64,"); got != 5 { + t.Fatalf("expected 5 inlined copies, got %d:\n%s", got, out) + } + if log.count() != 1 { + t.Fatalf("duplicate src downloaded %d times, want 1", log.count()) + } +} + +func TestAssetResolverRespectsPerPageImageCap(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + var sb strings.Builder + total := maxPageImages + 5 + for i := 0; i < total; i++ { + sb.WriteString(``) + } + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String()) + + if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages { + t.Fatalf("inlined %d images, want cap %d", got, maxPageImages) + } + if log.count() != maxPageImages { + t.Fatalf("downloaded %d images, want cap %d", log.count(), maxPageImages) + } + if got := strings.Count(out, "/wiki/download/attachments"); got != 5 { + t.Fatalf("expected 5 images past the cap to keep original src, got %d", got) + } +} + +func TestAssetResolverCapsOccurrencesNotUniqueDownloads(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + // Two unique URLs but 35 occurrences. Counting the cap by unique URL would + // download cold.png too and inline 30 + 5 data URIs, overflowing the downstream + // per-page limit that counts every occurrence. The budget must be occurrences. + hot := `` + cold := `` + in := strings.Repeat(hot, maxPageImages) + strings.Repeat(cold, 5) + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + + if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages { + t.Fatalf("inlined %d data URIs, want cap %d", got, maxPageImages) + } + if got := strings.Count(out, "/wiki/download/attachments/1/cold.png"); got != 5 { + t.Fatalf("expected the 5 images past the cap to keep original src, got %d", got) + } + // The repeated image downloads once; the image past the budget never downloads. + if log.count() != 1 { + t.Fatalf("downloaded %d images, want 1 (dedup + no over-budget fetch): %v", log.count(), log.all()) + } + if log.hasPath("/wiki/download/attachments/1/cold.png") { + t.Fatalf("image past the occurrence cap was downloaded: %v", log.all()) + } +} + +// largeTestImage is just under maxImageBytes, so a handful of them overflow +// maxPageInlineBytes long before the per-page occurrence cap. +var largeTestImage = bytes.Repeat([]byte("a"), int(maxImageBytes)-1024) + +func TestAssetResolverStopsInliningPastPageByteBudget(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "image/png", largeTestImage), nil + }) + want := maxPageInlineBytes / len(testDataURI(largeTestImage)) + total := want + 3 + src := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), strings.Repeat(src, total)) + + if got := strings.Count(out, "data:image/png;base64,"); got != want { + t.Fatalf("inlined %d copies, want %d within the %d-byte page budget", got, want, maxPageInlineBytes) + } + if got := strings.Count(out, "/wiki/download/attachments/1/big.png"); got != total-want { + t.Fatalf("expected %d copies past the budget to keep original src, got %d", total-want, got) + } +} + +func TestAssetResolverDownloadCacheRespectsPageByteBudget(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "image/png", largeTestImage), nil + }) + want := maxPageInlineBytes / len(testDataURI(largeTestImage)) + urls := make([]string, 0, want+3) + for i := 0; i < want+3; i++ { + urls = append(urls, testBaseURL+"/download/attachments/1/img"+strings.Repeat("x", i+1)+".png") + } + cache := newAssetResolver(newImageTestClient(rt)).downloadAll(context.Background(), urls) + + if len(cache) != want { + t.Fatalf("cached %d images, want %d within the %d-byte page budget", len(cache), want, maxPageInlineBytes) + } +} + +func TestAssetResolverSharesCapWithExistingDataURIImages(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + // Data URI images already in the page share the downstream 30-image budget with + // the ones we generate. With 10 existing, only 20 of the 25 private images may + // be inlined; the rest keep their original URL instead of overflowing the cap. + var sb strings.Builder + for i := 0; i < 10; i++ { + sb.WriteString(``) + } + for i := 0; i < 25; i++ { + sb.WriteString(``) + } + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String()) + + // The 10 pre-existing data URIs are preserved untouched. + if got := strings.Count(out, "data:image/gif;base64,EXISTING"); got != 10 { + t.Fatalf("preserved %d existing data URIs, want 10", got) + } + // Exactly 20 private images inlined: 30 shared budget - 10 already present. + if got := strings.Count(out, testDataURI(testPNG)); got != 20 { + t.Fatalf("inlined %d private images, want 20", got) + } + // The 5 private images past the shared budget keep their original URL. + if got := strings.Count(out, "/wiki/download/attachments"); got != 5 { + t.Fatalf("expected 5 private images past the cap to keep original src, got %d", got) + } + // Only the 20 inlinable private images are downloaded. + if log.count() != 20 { + t.Fatalf("downloaded %d images, want 20: %v", log.count(), log.all()) + } +} + +func TestAssetResolverIgnoresDataSrcAttribute(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + // data-src must not be mistaken for src; there is no real src here. + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if out != in { + t.Fatalf("data-src was treated as src:\n%s", out) + } + if log.count() != 0 { + t.Fatalf("data-src triggered a download: %v", log.all()) + } +} + +func TestAssetResolverToleratesGreaterThanInAttribute(t *testing.T) { + log := newReqLog() + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + log.add(req) + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `a > b` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if !strings.Contains(out, testDataURI(testPNG)) { + t.Fatalf("image with '>' in alt was not inlined:\n%s", out) + } + if !strings.Contains(out, `alt="a > b"`) { + t.Fatalf("alt attribute was corrupted:\n%s", out) + } +} + +func TestAssetResolverUnescapesAmpersandInQuery(t *testing.T) { + var gotPath string + var gotQuery string + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + gotPath = req.URL.Path + gotQuery = req.URL.RawQuery + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + in := `` + out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in) + if !strings.Contains(out, "data:image/png;base64,") { + t.Fatalf("image with & query was not inlined:\n%s", out) + } + if gotPath != "/wiki/download/attachments/1/a.png" || gotQuery != "api=v2&v=3" { + t.Fatalf("query not unescaped: path=%q query=%q", gotPath, gotQuery) + } +} + +func TestDownloadPrivateImageLimitsBodySize(t *testing.T) { + big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1) + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "image/png", big), nil + }) + c := newImageTestClient(rt) + _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/big.png") + if !errors.Is(err, errImageTooLarge) { + t.Fatalf("downloadImage() error = %v, want errImageTooLarge", err) + } +} + +func TestDownloadPrivateImageValidatesContentType(t *testing.T) { + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "application/json", []byte(`{"error":"unauthorized"}`)), nil + }) + c := newImageTestClient(rt) + _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png") + if !errors.Is(err, errImageNonImageContent) { + t.Fatalf("downloadImage() error = %v, want errImageNonImageContent", err) + } +} + +func TestDownloadPrivateImageSniffsWhenContentTypeMissing(t *testing.T) { + png := append([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, bytes.Repeat([]byte{0}, 64)...) + rt := roundTripper(func(_ *http.Request) (*http.Response, error) { + return imageResp(http.StatusOK, "", png), nil + }) + c := newImageTestClient(rt) + body, mime, err := c.downloadImage( + context.Background(), + "https://confluence.test/wiki/download/attachments/1/a.png", + ) + if err != nil { + t.Fatalf("downloadImage() error = %v", err) + } + if mime != "image/png" { + t.Fatalf("sniffed mime = %q, want image/png", mime) + } + if !bytes.Equal(body, png) { + t.Fatalf("body mismatch: %d bytes", len(body)) + } +} + +func TestDownloadPrivateImageRejectsCrossOriginRedirect(t *testing.T) { + rt := roundTripper(func(req *http.Request) (*http.Response, error) { + if req.URL.Host == "confluence.test" { + return redirectResp("https://evil.example.com/a.png"), nil + } + return imageResp(http.StatusOK, "image/png", testPNG), nil + }) + c := newImageTestClient(rt) + _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png") + if !errors.Is(err, errImageRedirectOffOrigin) { + t.Fatalf("downloadImage() error = %v, want errImageRedirectOffOrigin", err) + } +} + +// TestFetchStreamInlinesPrivateImageAsDataURI is the connector integration test: +// a body.view containing a same-origin private image flows through FetchStream +// and the emitted FetchedItem.Content carries an inline data URI (which the +// downstream docparser pipeline later persists and rewrites to resource://). +func TestFetchStreamInlinesPrivateImageAsDataURI(t *testing.T) { + api := &streamAPI{ + pages: []streamPage{{id: "p1", title: "Page", version: 1}}, + bodyHTML: map[string]string{"p1": `

hi

`}, + } + h := &captureHandler{} + if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), nil, h); err != nil { + t.Fatal(err) + } + if len(h.items) != 1 { + t.Fatalf("emitted %d items, want 1", len(h.items)) + } + content := string(h.items[0].Content) + if !strings.Contains(content, "data:image/png;base64,") { + t.Fatalf("private image was not inlined into Content:\n%s", content) + } + if strings.Contains(content, "/wiki/download/attachments") { + t.Fatalf("original private Confluence URL leaked into Content:\n%s", content) + } + if api.imageCalls != 1 { + t.Fatalf("image download calls = %d, want 1", api.imageCalls) + } +} diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go index ec156e38..66fa29ab 100644 --- a/internal/datasource/connector/confluence/client.go +++ b/internal/datasource/connector/confluence/client.go @@ -17,19 +17,29 @@ import ( ) const ( - maxJSONResponseBytes int64 = 20 << 20 - requestAttempts = 4 - maxErrorResponseRunes = 1000 - maxRetryDelay = 60 * time.Second - maxPaginationHops = 10000 + maxJSONResponseBytes int64 = 20 << 20 + maxImageBytes int64 = 10 << 20 + maxImageRedirects = 10 + requestAttempts = 4 + maxErrorResponseRunes = 1000 + maxRetryDelay = 60 * time.Second + maxPaginationHops = 10000 + maxTraversalNodes = 100000 + maxTransparentTraversalNodes = 10000 + maxTransparentTraversalDepth = 32 // maxEmptyServerPages bounds how far a Server/Data Center page listing - // follows _links.next through empty pages. Some builds keep returning - // next past the end of a space (#3596); permission filtering can also - // empty one page mid-listing, so a single empty page is not the end. + // follows _links.next through empty pages. maxEmptyServerPages = 3 - // maxDeletionProbes caps how many unlisted pages one sync looks up - // individually after a cut-short listing; the rest wait for the next run. - maxDeletionProbes = 200 + maxDeletionProbes = 200 +) + +var ( + errImageTooLarge = errors.New("confluence image exceeds size limit") + errImageNonImageContent = errors.New("confluence image response is not an image") + // errImageRedirectOffOrigin fires whenever a redirect target leaves the base + // URL boundary resolveEndpoint enforces — a different origin, or the same host + // but a path outside the configured context path (e.g. /wiki → /other-app). + errImageRedirectOffOrigin = errors.New("confluence image redirect left the configured base URL") ) type apiError struct { @@ -232,6 +242,121 @@ func (c *client) resourceURL(link string) string { return resolved } +// sameOrigin reports whether u shares scheme and host with the configured base +// URL. Only same-origin URLs may carry Confluence credentials. +func (c *client) sameOrigin(u *url.URL) bool { + if u == nil { + return false + } + base, err := url.Parse(c.cfg.baseURL) + if err != nil { + return false + } + return strings.EqualFold(base.Scheme, u.Scheme) && strings.EqualFold(base.Host, u.Host) +} + +// withinImageScope reports whether u stays inside the boundary resolveEndpoint +// enforces for the initial image URL: the same origin AND a path inside the +// configured context path. Redirect targets are validated with this same rule, +// not merely sameOrigin, because Go forwards Basic Auth on a same-host redirect — +// so a hop like /wiki/download → /other-app/image must fail closed too. +func (c *client) withinImageScope(u *url.URL) bool { + if u == nil { + return false + } + _, err := c.resolveEndpoint(u.String()) + return err == nil +} + +// downloadImage fetches a same-origin private image using the connector's +// credentials. It streams with a hard size cap, fails closed if a redirect left +// the Confluence origin, and validates the response really is an image. It is a +// best-effort single attempt: a failed image degrades to keeping its original +// URL rather than retrying or failing the page. +func (c *client) downloadImage(ctx context.Context, absURL string) ([]byte, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, absURL, nil) + if err != nil { + return nil, "", err + } + req.SetBasicAuth(c.cfg.username, c.cfg.secret) + req.Header.Set("Accept", "image/*,*/*;q=0.8") + resp, err := c.imageHTTPClient().Do(req) + if err != nil { + return nil, "", err + } + defer func() { _ = resp.Body.Close() }() + // Backstop for the redirect policy: imageHTTPClient's CheckRedirect already + // refuses any hop that leaves the base before it is followed, so this only + // re-verifies the final URL when the transport populated resp.Request. A custom + // RoundTripper may leave it nil, in which case there is no followed redirect to + // re-check and the download is allowed to proceed. + if resp.Request != nil && !c.withinImageScope(resp.Request.URL) { + return nil, "", errImageRedirectOffOrigin + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, "", &apiError{endpoint: "image", status: resp.StatusCode} + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxImageBytes+1)) + if err != nil { + return nil, "", err + } + if int64(len(body)) > maxImageBytes { + return nil, "", errImageTooLarge + } + mime, ok := imageContentType(resp.Header.Get("Content-Type"), body) + if !ok { + return nil, "", errImageNonImageContent + } + return body, mime, nil +} + +// imageHTTPClient reuses the connector's SSRF-safe transport and timeout but +// pins redirects to the configured Confluence base, so a private-image fetch can +// never be bounced to a third party or to another app on the same host. The +// redirect check fails closed: any hop that resolveEndpoint would reject — a +// foreign origin, or a same-host path outside the context path (which Go would +// still send Basic Auth to) — is refused outright rather than merely stripped of +// credentials. +func (c *client) imageHTTPClient() *http.Client { + return &http.Client{ + Transport: c.http.Transport, + Timeout: c.http.Timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= maxImageRedirects { + return errors.New("confluence image stopped after too many redirects") + } + if !c.withinImageScope(req.URL) { + return errImageRedirectOffOrigin + } + return nil + }, + } +} + +// imageContentType normalizes a response Content-Type to an image MIME type. An +// empty or generic binary type falls back to content sniffing; any explicit +// non-image type (for example the text/html login page Confluence returns when a +// token expired) is rejected so it is never stored as an image. +func imageContentType(header string, body []byte) (string, bool) { + ct := strings.ToLower(strings.TrimSpace(header)) + if i := strings.Index(ct, ";"); i >= 0 { + ct = strings.TrimSpace(ct[:i]) + } + switch { + case strings.HasPrefix(ct, "image/"): + return ct, true + case ct != "" && ct != "application/octet-stream": + return "", false + } + if sniff := strings.ToLower(http.DetectContentType(body)); strings.HasPrefix(sniff, "image/") { + if i := strings.Index(sniff, ";"); i >= 0 { + sniff = strings.TrimSpace(sniff[:i]) + } + return sniff, true + } + return "", false +} + func (c *client) ping(ctx context.Context) error { if c.cfg.cloud() { return c.get(ctx, "/api/v2/spaces?limit=1", nil) @@ -308,10 +433,6 @@ func withCloudPageQuery(next string) string { return parsed.String() } -// pages lists every current page in a space. complete is false when a Server -// listing was cut short after maxEmptyServerPages empty pages that still -// linked onward; the caller must not treat pages missing from such a listing -// as deleted. func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) { if c.cfg.cloud() { var all []page @@ -371,10 +492,6 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo return all, complete, err } -// pageInSpace reports whether a Server page is still a current page of the -// space. Confluence answers 404 for a page that was deleted or moved to the -// trash; a page that is no longer current or now belongs to another space is -// gone from this one too, matching what a complete listing would omit. func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) { var result struct { Status string `json:"status"` @@ -406,3 +523,349 @@ func (c *client) body(ctx context.Context, id string) (pageBody, error) { err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=body.view,version,space", &result) return result, err } + +// pageDetail obtains identity, ownership, and version without requiring a +// rendered body. Scope enumeration must remain complete when a body is +// temporarily unavailable, otherwise a transient content error would be +// indistinguishable from a missing page during deletion reconciliation. +func (c *client) pageDetail(ctx context.Context, id string) (page, error) { + var result page + if c.cfg.cloud() { + err := c.get(ctx, "/api/v2/pages/"+url.PathEscape(id), &result) + return result, err + } + err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=version,space", &result) + return result, err +} + +// validatePageOwnership performs the constant-cost check required before a +// picker expansion. Ancestors are deliberately not fetched here: they are only +// needed by ResolveResourceAncestors and buildSyncPlan. +func (c *client) validatePageOwnership(ctx context.Context, s space, pageID string) error { + full, err := c.pageDetail(ctx, pageID) + if err != nil { + return err + } + if c.cfg.cloud() { + if full.SpaceID != s.ID { + return fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID) + } + return nil + } + if full.Space.Key == "" || full.Space.Key != s.Key { + return fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + return nil +} + +func serverChildPagesEndpoint(pageID string) string { + query := url.Values{"expand": []string{serverPageExpand}, "limit": []string{"100"}} + return "/rest/api/content/" + url.PathEscape(pageID) + "/child/page?" + query.Encode() +} + +func serverTopLevelPagesEndpoint(spaceKey string) string { + query := url.Values{"depth": []string{"root"}, "expand": []string{serverPageExpand}, "limit": []string{"100"}} + return "/rest/api/space/" + url.PathEscape(spaceKey) + "/content/page?" + query.Encode() +} + +func (c *client) listServerPages(ctx context.Context, endpoint string) ([]page, error) { + start, _ := url.Parse(endpoint) + startQuery := start.Query() + var all []page + err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) { + var result pageList + if err := c.get(ctx, pageURL, &result); err != nil { + return "", err + } + all = append(all, result.Results...) + next := result.Links.Next + if next != "" { + next = withServerListQuery(next, startQuery) + } + return next, nil + }) + return all, err +} + +// Server pagination links commonly retain only start/limit. Restore the query +// that defines the listing scope (notably depth=root) as well as expand. +func withServerListQuery(next string, required url.Values) string { + parsed, err := url.Parse(next) + if err != nil { + return next + } + query := parsed.Query() + for key, values := range required { + if query.Get(key) == "" && len(values) > 0 { + query[key] = values + } + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +func (c *client) listCloudHierarchy(ctx context.Context, endpoint string) ([]page, error) { + var all []page + err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) { + var result cloudHierarchyList + if err := c.get(ctx, pageURL, &result); err != nil { + return "", err + } + for _, item := range result.Results { + all = append( + all, + page{ + ID: item.ID, + Title: item.Title, + Status: item.Status, + Kind: item.Type, + SpaceID: item.SpaceID, + Links: item.Links, + }, + ) + } + return result.Links.Next, nil + }) + return all, err +} + +// topLevelPages retrieves one level only. The Server/DC depth=root endpoint +// is paginated by the server; it deliberately never falls back to scanning all +// pages because that would defeat lazy selection for large spaces. +// +// Cloud applies the same lazy contract: depth=0 returns only root-level pages, +// and Cloud exposes no endpoint to enumerate a space's top-level folders +// (CONFCLOUD-84275), so pages nested directly under those containers stay +// unselectable in the picker for now. Recovering them would require a +// whole-space scan per expand, which is exactly what lazy loading forbids; +// selecting the whole space still syncs them. The connector flags this +// limitation through resource metadata so the picker can surface it. +func (c *client) topLevelPages(ctx context.Context, s space) ([]page, error) { + var pages []page + var err error + if c.cfg.cloud() { + pages, err = c.listCloudHierarchy( + ctx, + "/api/v2/spaces/"+url.PathEscape(s.ID)+"/pages?status=current&depth=0&limit=250", + ) + if err == nil { + visible := pages[:0] + for _, page := range pages { + if page.Kind == "" || page.Kind == "page" { + visible = append(visible, page) + } + } + pages = visible + } + } else { + pages, err = c.listServerPages(ctx, serverTopLevelPagesEndpoint(s.Key)) + } + if err == nil { + return pages, nil + } + // The depth=root navigation hint is Server/DC-specific; a Cloud 400/404 has + // different causes and must surface the raw API error for diagnosis. + if !c.cfg.cloud() { + var api *apiError + if errors.As(err, &api) && (api.status == http.StatusBadRequest || api.status == http.StatusNotFound) { + return nil, fmt.Errorf( + "confluence server does not support complete top-level page navigation for this space; "+ + "select the Space to sync all pages: %w", + err, + ) + } + } + return nil, err +} + +func (c *client) directChildPages(ctx context.Context, _ space, pageID string) ([]page, error) { + if c.cfg.cloud() { + return c.visibleCloudPageChildren(ctx, pageID) + } + return c.listServerPages(ctx, serverChildPagesEndpoint(pageID)) +} + +func cloudDirectChildrenEndpoint(kind, id string) (string, error) { + plural := map[string]string{ + "page": "pages", "folder": "folders", "database": "databases", "whiteboard": "whiteboards", "embed": "embeds", + }[kind] + if plural == "" { + return "", fmt.Errorf("unsupported Confluence Cloud hierarchy type %q", kind) + } + return "/api/v2/" + plural + "/" + url.PathEscape(id) + "/direct-children?limit=250", nil +} + +// visibleCloudPageChildren projects Cloud's mixed-content hierarchy onto a +// Page-only picker. Containers remain connector-internal and are traversed +// until the next visible Page is found. +func (c *client) visibleCloudPageChildren(ctx context.Context, pageID string) ([]page, error) { + return c.walkCloudPageHierarchy(ctx, pageID, false) +} + +// walkCloudPageHierarchy traverses Cloud's mixed content tree through +// direct-children endpoints. Picker expansion stops at visible pages, while a +// sync walk continues through them to return the complete page subtree. +func (c *client) walkCloudPageHierarchy(ctx context.Context, pageID string, descendPages bool) ([]page, error) { + type queuedNode struct { + id, kind string + depth int + } + queue := []queuedNode{{id: pageID, kind: "page"}} + visited := map[string]struct{}{"page:" + pageID: {}} + pages := make([]page, 0) + nodeLimit := maxTransparentTraversalNodes + if descendPages { + nodeLimit = maxTraversalNodes + } + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + endpoint, err := cloudDirectChildrenEndpoint(current.kind, current.id) + if err != nil { + return nil, err + } + children, err := c.listCloudHierarchy(ctx, endpoint) + if err != nil { + return nil, err + } + for _, child := range children { + kind := child.Kind + if kind == "" { + kind = "page" + } + key := kind + ":" + child.ID + if child.ID == "" || (kind == "page" && child.ID == pageID) { + continue + } + if _, exists := visited[key]; exists { + continue + } + if len(visited) >= nodeLimit { + return nil, fmt.Errorf( + "confluence Cloud hierarchy traversal exceeds %d nodes", + nodeLimit, + ) + } + visited[key] = struct{}{} + if kind == "page" { + pages = append(pages, child) + if !descendPages { + continue + } + } + depth := 0 + if kind != "page" { + depth = current.depth + 1 + if current.kind == "page" { + depth = 1 + } + } + if depth > maxTransparentTraversalDepth { + return nil, fmt.Errorf( + "confluence Cloud transparent traversal exceeds depth %d", + maxTransparentTraversalDepth, + ) + } + if _, err := cloudDirectChildrenEndpoint(kind, child.ID); err != nil { + return nil, err + } + queue = append(queue, queuedNode{id: child.ID, kind: kind, depth: depth}) + } + } + return pages, nil +} + +// pageAncestors validates that pageID belongs to s and returns its path from +// the space root. A selected page is fetched even if it is top-level, so a +// forged page:{space}:{page} ID cannot cross a space boundary. +func (c *client) pageAncestors(ctx context.Context, s space, pageID string) ([]page, error) { + if c.cfg.cloud() { + full, err := c.pageDetail(ctx, pageID) + if err != nil { + return nil, err + } + if full.SpaceID != s.ID { + return nil, fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID) + } + ancestors, err := c.listCloudHierarchy(ctx, "/api/v2/pages/"+url.PathEscape(pageID)+"/ancestors?limit=250") + if err != nil { + return nil, err + } + return ancestors, nil + } + var full page + if err := c.get(ctx, "/rest/api/content/"+url.PathEscape(pageID)+"?expand=ancestors,space", &full); err != nil { + return nil, err + } + if full.Space.Key == "" || full.Space.Key != s.Key { + return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + return full.Ancestors, nil +} + +// pageSubtree uses direct-child traversal for Server/DC because that endpoint +// is stable across supported versions. Cloud uses the same complete +// mixed-content hierarchy walk as the picker, but continues through visible +// pages to include every page in the selected subtree. +func (c *client) pageSubtree(ctx context.Context, s space, pageID string) ([]page, error) { + full, err := c.pageDetail(ctx, pageID) + if err != nil { + return nil, err + } + if c.cfg.cloud() { + if full.SpaceID != s.ID { + return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + } else if full.Space.Key == "" || full.Space.Key != s.Key { + return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID) + } + root := full + if c.cfg.cloud() { + descendants, err := c.walkCloudPageHierarchy(ctx, pageID, true) + if err != nil { + return nil, err + } + out := []page{root} + for _, descendant := range descendants { + // Direct-child responses are intentionally compact. Hydrate each page + // before version comparison; an unknown version must never be treated + // as an unchanged stable token. + detail, err := c.pageDetail(ctx, descendant.ID) + if err != nil { + return nil, err + } + if detail.SpaceID != s.ID { + return nil, fmt.Errorf("page %s does not belong to space %s", descendant.ID, s.ID) + } + out = append(out, detail) + } + return out, nil + } + queue, out := []page{root}, []page{root} + seen := map[string]bool{root.ID: true} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + children, err := c.directChildPages(ctx, s, current.ID) + if err != nil { + return nil, err + } + for _, child := range children { + if child.ID == "" || seen[child.ID] { + continue + } + if len(seen) >= maxTraversalNodes { + return nil, fmt.Errorf("confluence page subtree exceeds %d nodes", maxTraversalNodes) + } + seen[child.ID] = true + if child.SpaceID != "" && child.SpaceID != s.ID { + return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID) + } + if !c.cfg.cloud() && child.Space.Key != "" && child.Space.Key != s.Key { + return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID) + } + queue, out = append(queue, child), append(out, child) + } + } + return out, nil +} diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go index 1a7df281..1113a35d 100644 --- a/internal/datasource/connector/confluence/client_pages_test.go +++ b/internal/datasource/connector/confluence/client_pages_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -145,6 +146,70 @@ func TestServerSpacePageListPrefersFlatResults(t *testing.T) { } } +func TestTopLevelPagesServerRestoresScopeOnNext(t *testing.T) { + var depths, expands []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rest/api/space/ENG/content/page" { + http.NotFound(w, r) + return + } + depths = append(depths, r.URL.Query().Get("depth")) + expands = append(expands, r.URL.Query().Get("expand")) + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("start") == "100" { + _, _ = w.Write([]byte(`{ + "results": [{"id": "2", "title": "Second", "version": {"number": 2}}] + }`)) + return + } + _, _ = w.Write([]byte(`{ + "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}, + "results": [{"id": "1", "title": "Root", "version": {"number": 1}}] + }`)) + })) + t.Cleanup(server.Close) + + c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} + pages, err := c.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + if err != nil { + t.Fatal(err) + } + if len(pages) != 2 || pages[0].ID != "1" || pages[1].ID != "2" { + t.Fatalf("pages = %#v", pages) + } + // The server's _links.next drops depth and expand; the connector must + // rebuild the listing scope or later pages would look versionless. + if len(depths) != 2 || depths[0] != "root" || depths[1] != "root" { + t.Fatalf("depth query = %#v", depths) + } + if len(expands) != 2 || expands[0] != serverPageExpand || expands[1] != serverPageExpand { + t.Fatalf("expand query = %#v", expands) + } +} + +func TestTopLevelPagesCloudSurfacesRawAPIError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"message":"space not found"}`)) + })) + t.Cleanup(server.Close) + + cloud := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()} + _, err := cloud.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + if err == nil { + t.Fatal("cloud topLevelPages unexpectedly succeeded") + } + if strings.Contains(err.Error(), "does not support") { + t.Fatalf("cloud 404 reused the Server/DC navigation hint: %v", err) + } + + srv := &client{cfg: config{baseURL: server.URL + "/wiki"}, http: server.Client()} + _, err = srv.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"}) + if err == nil || !strings.Contains(err.Error(), "confluence server does not support") { + t.Fatalf("server error = %v; want the Server/DC navigation hint", err) + } +} + func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) { var depths []string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go index fe2a6f5c..083d5bf1 100644 --- a/internal/datasource/connector/confluence/connector.go +++ b/internal/datasource/connector/confluence/connector.go @@ -3,10 +3,14 @@ package confluence import ( "context" + "crypto/sha256" "errors" "fmt" "net/http" + "sort" "strings" + "sync" + "time" htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/Tencent/WeKnora/internal/datasource" @@ -21,12 +25,64 @@ var ( // Connector implements datasource.StreamingConnector for Confluence. type Connector struct { - newClient func(config) (*client, error) + newClient func(config) (*client, error) + spaceCache spaceCache +} + +const pickerSpaceCacheTTL = time.Minute + +type spaceCache struct { + mu sync.Mutex + entries map[string]cachedSpaces +} + +type cachedSpaces struct { + values []space + expires time.Time } // NewConnector creates a Confluence connector. func NewConnector() *Connector { return &Connector{newClient: newClient} } +func (c *Connector) pickerSpaces(ctx context.Context, client *client, cfg config) ([]space, error) { + key := pickerSpaceCacheKey(cfg) + now := time.Now() + c.spaceCache.mu.Lock() + if cached, ok := c.spaceCache.entries[key]; ok && now.Before(cached.expires) { + values := append([]space(nil), cached.values...) + c.spaceCache.mu.Unlock() + return values, nil + } + c.spaceCache.mu.Unlock() + spaces, err := client.spaces(ctx) + if err != nil { + return nil, err + } + c.spaceCache.mu.Lock() + if c.spaceCache.entries == nil { + c.spaceCache.entries = make(map[string]cachedSpaces) + } + for key, cached := range c.spaceCache.entries { + if !now.Before(cached.expires) { + delete(c.spaceCache.entries, key) + } + } + c.spaceCache.entries[key] = cachedSpaces{ + values: append([]space(nil), spaces...), + expires: now.Add(pickerSpaceCacheTTL), + } + c.spaceCache.mu.Unlock() + return spaces, nil +} + +// pickerSpaceCacheKey partitions cached space visibility by every credential +// component that can affect permissions. The secret is fingerprinted so it +// never becomes a map key retained in process memory or appears in diagnostics. +func pickerSpaceCacheKey(cfg config) string { + fingerprint := sha256.Sum256([]byte(cfg.secret)) + return cfg.edition + "\x00" + cfg.baseURL + "\x00" + cfg.username + "\x00" + fmt.Sprintf("%x", fingerprint) +} + // Type returns the connector type identifier. func (*Connector) Type() string { return types.ConnectorTypeConfluence } @@ -52,13 +108,95 @@ func (c *Connector) Validate(ctx context.Context, ds *types.DataSourceConfig) er return client.ping(ctx) } -// ListResources returns the Confluence spaces the credentials can see. +// ListResources exposes a lazy Space → Page tree. It never scans a whole +// space merely to render the picker. func (c *Connector) ListResources( ctx context.Context, ds *types.DataSourceConfig, parentID string, ) ([]types.Resource, error) { - if parentID != "" { - return nil, nil + client, cfg, err := c.configured(ds) + if err != nil { + return nil, err } + if parentID == "" { + spaces, err := c.pickerSpaces(ctx, client, cfg) + if err != nil { + return nil, err + } + out := make([]types.Resource, 0, len(spaces)) + for _, s := range spaces { + metadata := map[string]interface{}{"space_key": s.Key} + if cfg.cloud() { + // Cloud cannot enumerate a space's top-level folders, so pages under + // them are unselectable in the picker; the frontend surfaces this + // limitation when the space is expanded. + metadata["hierarchy_limitation"] = "cloud_top_level_containers" + } + out = append(out, types.Resource{ + ExternalID: s.ID, Name: s.Name, Type: "space", + URL: client.resourceURL(s.Links.WebUI), HasChildren: true, + Metadata: metadata, + }) + } + return out, nil + } + ref, err := parseResourceID(parentID) + if err != nil { + return nil, err + } + spaces, err := c.pickerSpaces(ctx, client, cfg) + if err != nil { + return nil, err + } + var selected space + found := false + for _, s := range spaces { + if s.ID == ref.SpaceID { + selected, found = s, true + break + } + } + if !found { + return nil, fmt.Errorf("confluence space %s is unavailable", ref.SpaceID) + } + var pages []page + if ref.Kind == resourceSpace { + pages, err = client.topLevelPages(ctx, selected) + } else { + if ref.Kind == resourcePage { + if err := client.validatePageOwnership(ctx, selected, ref.PageID); err != nil { + return nil, fmt.Errorf("validate Confluence page %s: %w", ref.PageID, err) + } + } + pages, err = client.directChildPages(ctx, selected, ref.PageID) + } + if err != nil { + return nil, fmt.Errorf("list Confluence resources under %s: %w", parentID, err) + } + out := make([]types.Resource, 0, len(pages)) + for _, p := range pages { + out = append(out, pageResource(client, selected, parentID, p)) + } + return out, nil +} + +func pageResource(client *client, s space, parentID string, p page) types.Resource { + return types.Resource{ + ExternalID: makePageResourceID(s.ID, p.ID), ParentID: parentID, Name: p.Title, Type: "page", + URL: client.resourceURL(p.Links.WebUI), HasChildren: true, + Metadata: map[string]interface{}{"space_id": s.ID, "page_id": p.ID}, + } +} + +// ResolveResourceAncestors expands only the O(depth) path required to reveal +// persisted page selections in the lazy picker. Resolution is best effort: +// a deleted page, an unavailable space, or a malformed id skips just its own +// reveal path instead of failing every saved selection; the sync path keeps +// its strict validation in buildSyncPlan. +func (c *Connector) ResolveResourceAncestors( + ctx context.Context, + ds *types.DataSourceConfig, + resourceIDs []string, +) ([]string, error) { client, _, err := c.configured(ds) if err != nil { return nil, err @@ -67,25 +205,47 @@ func (c *Connector) ListResources( if err != nil { return nil, err } - out := make([]types.Resource, 0, len(spaces)) + byID := make(map[string]space, len(spaces)) for _, s := range spaces { - out = append(out, types.Resource{ - ExternalID: s.ID, - Name: s.Name, - Type: "space", - URL: client.resourceURL(s.Links.WebUI), - HasChildren: false, - Metadata: map[string]interface{}{"space_key": s.Key}, - }) + byID[s.ID] = s + } + seen, out := map[string]bool{}, []string{} + add := func(id string) { + if id != "" && !seen[id] { + seen[id] = true + out = append(out, id) + } + } + for _, id := range resourceIDs { + ref, err := parseResourceID(id) + if err != nil { + logger.Warnf(ctx, "skip malformed Confluence resource %q during ancestor resolution: %v", id, err) + continue + } + if ref.Kind == resourceSpace { + continue + } + s, ok := byID[ref.SpaceID] + if !ok { + logger.Warnf(ctx, "skip Confluence resource %s: space %s is unavailable", id, ref.SpaceID) + continue + } + ancestors, err := client.pageAncestors(ctx, s, ref.PageID) + if err != nil { + logger.Warnf(ctx, "skip Confluence resource %s during ancestor resolution: %v", id, err) + continue + } + add(s.ID) + for _, ancestor := range ancestors { + if ancestor.Kind != "" && ancestor.Kind != "page" { + continue + } + add(makePageResourceID(s.ID, ancestor.ID)) + } } return out, nil } -// ResolveResourceAncestors is a no-op: Confluence spaces are a flat list. -func (*Connector) ResolveResourceAncestors(context.Context, *types.DataSourceConfig, []string) ([]string, error) { - return nil, nil -} - // FetchAll syncs every selected space. Deletion reconciliation needs FetchStream. func (c *Connector) FetchAll( ctx context.Context, ds *types.DataSourceConfig, resourceIDs []string, @@ -158,50 +318,89 @@ func (c *Connector) fetchStream( if err != nil { return nil, err } - spaces, err := client.spaces(ctx) + plan, err := c.buildSyncPlan(ctx, client, ds.ResourceIDs) if err != nil { - return nil, fmt.Errorf("list Confluence spaces: %w", err) + return nil, err } - byID := make(map[string]space, len(spaces)) - for _, s := range spaces { - byID[s.ID] = s + currentRoots := make(map[string]bool, len(plan.Roots)) + for _, root := range plan.Roots { + currentRoots[root.ResourceID] = true } baseline, next := prepareSyncCursors(old, forceFull) - for _, resourceID := range ds.ResourceIDs { - s, found := byID[resourceID] - if !found { - return nil, fmt.Errorf( - "selected Confluence space %s is unavailable; refusing destructive reconciliation", - resourceID, - ) + // A prior successful body is reusable when ownership moves to a different + // selected root and this run cannot fetch the new body. Keep that version in + // the new root rather than dropping it during ownership reconciliation. + priorVersionByPage := make(map[string]string) + for _, pages := range baseline.SpacePages { + for id, version := range pages { + if _, exists := priorVersionByPage[id]; !exists { + priorVersionByPage[id] = version + } + } + } + // A page may be returned by multiple overlapping roots (for example during + // a configuration change). Fetch it once, while retaining a cursor entry + // per scope root so existing cursor JSON remains compatible. + seenPages := make(map[string]struct{}) + incompleteRoots := make(map[string]bool) + incompleteSpaces := make([]space, 0) + for _, root := range plan.Roots { + resourceID, s := root.ResourceID, root.Space + var pages []page + complete := true + if root.Kind == syncWholeSpace { + pages, complete, err = client.pages(ctx, s) + } else { + pages, err = client.pageSubtree(ctx, s, root.PageID) } - pages, complete, err := client.pages(ctx, s) if err != nil { - return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err) + return nil, fmt.Errorf("list pages in Confluence scope %s: %w", resourceID, err) } + priorPages := baseline.SpacePages[resourceID] if !complete { - logger.Warnf(ctx, - "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+ - "listed %d pages, checking unlisted ones one by one before deleting", - s.Key, maxEmptyServerPages, len(pages)) + logger.Warnf( + ctx, + "[Confluence] space %s listing was cut short; checking unlisted pages before deleting", + s.Key, + ) + incompleteRoots[resourceID] = true + incompleteSpaces = append(incompleteSpaces, s) } - priorPages, hadBaseline := baseline.SpacePages[resourceID] if next.SpacePages[resourceID] == nil { next.SpacePages[resourceID] = map[string]string{} } - seen := make(map[string]struct{}, len(pages)) + if complete && len(pages) == 0 && len(priorPages) > 0 { + return nil, fmt.Errorf( + "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline", + s.Key, len(priorPages), + ) + } for _, summary := range pages { if err := ctx.Err(); err != nil { return nil, err } - seen[summary.ID] = struct{}{} - version := pageVersion(summary) - if forceFull { - if next.SpacePages[resourceID][summary.ID] == version { + if _, duplicate := seenPages[summary.ID]; duplicate { + // The first scope already owns the emitted document; preserve this + // root's version record so a later selection change stays incremental. + if !forceFull { + if version, ok := priorPages[summary.ID]; ok { + next.SpacePages[resourceID][summary.ID] = version + } + } + continue + } + seenPages[summary.ID] = struct{}{} + version, versionKnown := pageVersion(summary) + // An unknown version cannot prove a page unchanged, so it must + // always refresh the body instead of skipping on a stable "t:". + if versionKnown { + if forceFull { + if next.SpacePages[resourceID][summary.ID] == version { + continue + } + } else if priorPages[summary.ID] == version { continue } - } else if priorPages[summary.ID] == version { - continue } full, err := client.body(ctx, summary.ID) if err != nil { @@ -211,13 +410,23 @@ func (c *Connector) fetchStream( if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil { return nil, emitErr } + if !forceFull { + if prior, ok := priorVersionByPage[summary.ID]; ok { + next.SpacePages[resourceID][summary.ID] = prior + } + } continue } - item, err := markdownItem(client, resourceID, summary, full) + item, err := markdownItem(ctx, client, resourceID, summary, full) if err != nil { if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil { return nil, emitErr } + if !forceFull { + if prior, ok := priorVersionByPage[summary.ID]; ok { + next.SpacePages[resourceID][summary.ID] = prior + } + } continue } if err := h.Emit(ctx, item); err != nil { @@ -228,101 +437,119 @@ func (c *Connector) fetchStream( return nil, err } } - if !complete { - if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil { - return nil, err - } - } else if hadBaseline { - if len(pages) == 0 && len(priorPages) > 0 { - return nil, fmt.Errorf( - "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline", - s.Key, len(priorPages), - ) - } - missing := 0 - for id := range priorPages { - if _, exists := seen[id]; !exists { - missing++ - } - } - if missing >= 20 && missing*100 >= len(priorPages)*80 { - return nil, fmt.Errorf( - "refusing Confluence mirror deletion in %s: %d/%d pages disappeared", - s.Key, missing, len(priorPages), - ) - } - for id := range priorPages { - if _, exists := seen[id]; exists { - continue - } - if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil { - return nil, err - } + } + // Reconcile previous owners only after every current scope has been enumerated. + // A page may move between roots, so a missing page must be checked against every + // scope whose listing was cut short before a tombstone can be emitted. + previousPages := make(map[string]string) + for owner, pages := range baseline.SpacePages { + for id := range pages { + if _, exists := previousPages[id]; !exists { + previousPages[id] = owner } } } + missing := make([]string, 0) + probeCount := 0 + for id := range previousPages { + if _, present := seenPages[id]; present { + continue + } + owner := previousPages[id] + for _, s := range incompleteSpaces { + if probeCount >= maxDeletionProbes { + return nil, fmt.Errorf( + "Confluence deletion reconciliation deferred: more than %d pages need verification", + maxDeletionProbes, + ) + } + probeCount++ + present, err := client.pageInSpace(ctx, id, s.Key) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf( + "Confluence deletion reconciliation deferred: could not verify page %s in space %s: %w", + id, s.Key, err, + ) + } + if present { + version := baseline.SpacePages[owner][id] + if next.SpacePages[s.ID] == nil { + next.SpacePages[s.ID] = make(map[string]string) + } + next.SpacePages[s.ID][id] = version + seenPages[id] = struct{}{} + break + } + } + if _, present := seenPages[id]; present { + continue + } + missing = append(missing, id) + } + // Protect still-selected roots from a suspiciously large shrink. Removed + // roots are an explicit scope change, so their out-of-scope pages can be + // tombstoned once every current scope has been enumerated safely. + owners := make([]string, 0, len(baseline.SpacePages)) + for owner := range baseline.SpacePages { + owners = append(owners, owner) + } + sort.Strings(owners) + for _, owner := range owners { + // A deliberate scope change may remove most of an old root. The guard is + // only for unexpected shrinkage of a still-selected, completely listed root. + if !currentRoots[owner] || incompleteRoots[owner] { + continue + } + pages := baseline.SpacePages[owner] + missingInRoot := 0 + for id := range pages { + if _, present := seenPages[id]; !present { + missingInRoot++ + } + } + if len(pages) > 0 && missingInRoot >= 20 && missingInRoot*100 >= len(pages)*80 { + return nil, fmt.Errorf( + "refusing Confluence mirror deletion: %d/%d previously synced pages would be removed from scope %s", + missingInRoot, len(pages), owner, + ) + } + } + sort.Strings(missing) + for _, id := range missing { + deleted := types.FetchedItem{ + ExternalID: id, IsDeleted: true, SourceResourceID: previousPages[id], + Metadata: map[string]string{"channel": types.ChannelConfluence}, + } + if err := h.Emit(ctx, deleted); err != nil { + return nil, err + } + // Drop the tombstoned page before checkpointing so the cursor records + // the deletion itself; an interrupted run resumes with the remaining + // candidates instead of re-emitting completed tombstones. + removePageOwnership(&next, id) + if err := h.Checkpoint(ctx, next.syncCursor()); err != nil { + return nil, err + } + } + // Commit exactly the current ownership roots only after reconciliation. This + // removes cancelled scopes while keeping the on-disk cursor field compatible. + committedRoots := make(map[string]map[string]string, len(plan.Roots)) + for _, root := range plan.Roots { + if pages := next.SpacePages[root.ResourceID]; pages != nil { + committedRoots[root.ResourceID] = pages + } else { + committedRoots[root.ResourceID] = map[string]string{} + } + } + next.SpacePages = committedRoots next.FullSync = false next.FullSyncBaseline = nil return next.syncCursor(), nil } -// reconcileUnlisted handles the pages a cut-short listing did not return. The -// listing proves nothing about them, so each one is looked up on its own and -// deleted only when Confluence confirms it is gone from the space. Pages that -// are still there, could not be checked, or exceed the per-run probe budget -// stay tracked with their prior version and are checked again next run. -func reconcileUnlisted( - ctx context.Context, - c *client, - h datasource.StreamHandler, - s space, - resourceID string, - priorPages map[string]string, - seen map[string]struct{}, - next *cursor, -) error { - probed := 0 - for id, version := range priorPages { - if _, exists := seen[id]; exists { - continue - } - if probed >= maxDeletionProbes { - next.SpacePages[resourceID][id] = version - continue - } - probed++ - present, err := c.pageInSpace(ctx, id, s.Key) - if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err) - next.SpacePages[resourceID][id] = version - continue - } - if present { - next.SpacePages[resourceID][id] = version - continue - } - if err := emitDeleted(ctx, h, next, resourceID, id); err != nil { - return err - } - } - return nil -} - -func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error { - deleted := types.FetchedItem{ - ExternalID: id, IsDeleted: true, SourceResourceID: resourceID, - Metadata: map[string]string{"channel": types.ChannelConfluence}, - } - if err := h.Emit(ctx, deleted); err != nil { - return err - } - delete(next.SpacePages[resourceID], id) - return h.Checkpoint(ctx, next.syncCursor()) -} - func failedPageItem(resourceID string, summary page, err error) types.FetchedItem { code, reason := classifyConfluenceError(err) return types.FetchedItem{ @@ -361,8 +588,18 @@ func classifyConfluenceError(err error) (code, reason string) { return "confluence_sync_failed", "Confluence page could not be synced; see server logs" } -func markdownItem(client *client, resourceID string, summary page, full pageBody) (types.FetchedItem, error) { +func markdownItem( + ctx context.Context, + client *client, + resourceID string, + summary page, + full pageBody, +) (types.FetchedItem, error) { html := full.Body.View.Value + // Inline same-origin private Confluence images as data URIs so the generic + // docparser image pipeline can persist them and rewrite to resource:// URLs. + // Best-effort: a failed image keeps its original src and never fails the page. + html = newAssetResolver(client).Resolve(ctx, html) markdown, err := htmltomd.ConvertString(html) if err != nil { return types.FetchedItem{}, err diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go index 2f38c787..0197a55d 100644 --- a/internal/datasource/connector/confluence/connector_stream_test.go +++ b/internal/datasource/connector/confluence/connector_stream_test.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" "testing" + "time" "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/types" @@ -26,19 +27,18 @@ type streamPage struct { } type streamAPI struct { - cloud bool - pages []streamPage - bodyCalls int - bodyStatus map[string]int - listCalls int - // endlessNext mimics Server/Data Center builds that keep linking to a - // next page after the last one, returning empty results forever (#3596). - endlessNext bool - // probe answers the per-page existence check (GET content/{id}?expand=space) - // for pages that are not in the listing: an HTTP status, or 200 with the - // space key the page now belongs to. - probe map[string]probeReply - probeCalls int + cloud bool + pages []streamPage + bodyCalls int + bodyStatus map[string]int + bodyHTML map[string]string + imageCalls int + listCalls int + ancestorCalls int + spaceCalls int + endlessNext bool + probe map[string]probeReply + probeCalls int } type probeReply struct { @@ -71,7 +71,17 @@ func (a *streamAPI) pageMaps() []any { func (a *streamAPI) response(req *http.Request) (*http.Response, error) { path := req.URL.Path switch { + case strings.HasPrefix(path, "/wiki/download/"): + a.imageCalls++ + header := make(http.Header) + header.Set("Content-Type", "image/png") + return &http.Response{ + StatusCode: http.StatusOK, + Header: header, + Body: io.NopCloser(bytes.NewReader([]byte("private-png-bytes"))), + }, nil case path == "/wiki/rest/api/space": + a.spaceCalls++ return a.jsonResponse(map[string]any{ "results": []any{map[string]any{ "id": 1, "key": "ENG", "name": "Engineering", @@ -80,19 +90,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { }) case path == "/wiki/rest/api/space/ENG/content/page": a.listCalls++ - if !a.endlessNext { + if req.URL.Query().Get("depth") == "root" { + return a.jsonResponse(map[string]any{"results": a.pageMaps()}) + } + if a.endlessNext { + results := a.pageMaps() + if a.listCalls > 1 { + results = []any{} + } return a.jsonResponse(map[string]any{ - "results": a.pageMaps(), + "results": results, + "_links": map[string]any{ + "next": fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100), + }, }) } - results := a.pageMaps() - if a.listCalls > 1 { - results = []any{} - } - next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100) return a.jsonResponse(map[string]any{ - "results": results, - "_links": map[string]any{"next": next}, + "results": a.pageMaps(), }) case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space": a.probeCalls++ @@ -108,10 +122,36 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { Body: io.NopCloser(strings.NewReader("{}")), }, nil } - return a.jsonResponse(map[string]any{ - "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}, - }) + return a.jsonResponse( + map[string]any{"id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}}, + ) + case strings.HasPrefix(path, "/wiki/rest/api/content/") && strings.HasSuffix(path, "/child/page"): + return a.jsonResponse(map[string]any{"results": []any{}}) case strings.HasPrefix(path, "/wiki/rest/api/content/"): + if req.URL.Query().Get("expand") == "ancestors,space" { + a.ancestorCalls++ + return a.jsonResponse(map[string]any{ + "id": strings.TrimPrefix(path, "/wiki/rest/api/content/"), + "space": map[string]any{"key": "ENG", "name": "Engineering"}, + "ancestors": []any{}, + }) + } + if req.URL.Query().Get("expand") == "version,space" { + id := strings.TrimPrefix(path, "/wiki/rest/api/content/") + for _, page := range a.pages { + if page.id == id { + return a.jsonResponse( + map[string]any{ + "id": page.id, + "title": page.title, + "version": map[string]any{"number": page.version}, + "space": map[string]any{"key": "ENG", "name": "Engineering"}, + }, + ) + } + } + return nil, errors.New("missing page") + } if req.URL.Query().Get("expand") != "body.view,version,space" { return nil, errors.New("page body did not request rendered body.view") } @@ -137,8 +177,14 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { }) } return a.jsonResponse(map[string]any{"results": pages}) + case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/ancestors"): + return a.jsonResponse(map[string]any{"results": []any{}}) + case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/descendants"): + return a.jsonResponse(map[string]any{"results": []any{}}) + case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/direct-children"): + return a.jsonResponse(map[string]any{"results": []any{}}) case strings.HasPrefix(path, "/wiki/api/v2/pages/"): - if req.URL.Query().Get("body-format") != "view" { + if req.URL.Query().Get("body-format") != "" && req.URL.Query().Get("body-format") != "view" { return nil, errors.New("cloud page body did not request body-format=view") } return a.pageBody(strings.TrimPrefix(path, "/wiki/api/v2/pages/")) @@ -158,12 +204,17 @@ func (a *streamAPI) pageBody(id string) (*http.Response, error) { } for _, page := range a.pages { if page.id == id { + value := "

" + page.title + "

" + if custom, ok := a.bodyHTML[id]; ok { + value = custom + } return a.jsonResponse(map[string]any{ "id": page.id, "title": page.title, + "spaceId": "1", "version": map[string]any{"number": page.version, "by": map[string]any{"displayName": "Ada"}}, "space": map[string]any{"key": "ENG", "name": "Engineering"}, "_links": map[string]any{"webui": "/wiki/pages/" + page.id}, - "body": map[string]any{"view": map[string]any{"value": "

" + page.title + "

"}}, + "body": map[string]any{"view": map[string]any{"value": value}}, }) } } @@ -196,10 +247,11 @@ func cloudStreamConfig() *types.DataSourceConfig { } type captureHandler struct { - items []types.FetchedItem - checkpoints []*types.SyncCursor - emitErr error - succeedFirst int + items []types.FetchedItem + checkpoints []*types.SyncCursor + emitErr error + succeedFirst int + checkpointErrAt int } func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error { @@ -214,6 +266,9 @@ func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error { } func (h *captureHandler) Checkpoint(_ context.Context, c *types.SyncCursor) error { + if h.checkpointErrAt > 0 && len(h.checkpoints)+1 == h.checkpointErrAt { + return errors.New("checkpoint failed") + } raw, _ := json.Marshal(c) var stored types.SyncCursor _ = json.Unmarshal(raw, &stored) @@ -248,16 +303,144 @@ func TestFetchStreamSkipsUnchangedPages(t *testing.T) { } } +func TestFetchStreamUnknownVersionDoesNotSkipBody(t *testing.T) { + // version.number=0 with no when/createdAt leaves the version unknown; the + // legacy stable "t:" cursor value must not suppress the body refresh. + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 0}}} + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "t:"}) + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if api.bodyCalls != 1 || len(h.items) != 1 || h.items[0].IsDeleted { + t.Fatalf("unknown-version page fetched=%d items=%#v", api.bodyCalls, h.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "" { + t.Fatalf("cursor stored a stable unknown version: %#v", decoded) + } +} + func TestFetchStreamDetectsDeletedPages(t *testing.T) { api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} h := &captureHandler{} old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) - if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h); err != nil { + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { t.Fatal(err) } if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" { t.Fatalf("deletion items = %#v", h.items) } + if decoded := decodeCursor(next); decoded.SpacePages["1"]["p2"] != "" { + t.Fatalf("deleted page stayed in cursor as a ghost: %#v", decoded) + } +} + +func TestFetchStreamRemovesDeletedPageFromCursor(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" { + t.Fatalf("deletion items = %#v", h.items) + } + decoded := decodeCursor(next) + if decoded.SpacePages["1"]["p1"] != "v:1" { + t.Fatalf("surviving page missing from cursor: %#v", decoded) + } + for _, checkpoint := range h.checkpoints { + if stored := decodeCursor(checkpoint); stored.SpacePages["1"]["p2"] != "" { + t.Fatalf("checkpoint recorded a not-yet-deleted page as deleted: %#v", stored) + } + } + // A follow-up run must not re-emit the already tombstoned page. + followUp := &captureHandler{} + if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), next, followUp); err != nil { + t.Fatal(err) + } + for _, item := range followUp.items { + if item.IsDeleted { + t.Fatalf("ghost page re-tombstoned on the next run: %#v", followUp.items) + } + } +} + +func TestFetchStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"}) + first := &captureHandler{succeedFirst: 1} + if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, first); err == nil { + t.Fatal("interrupted tombstone run unexpectedly succeeded") + } + if len(first.items) != 1 || !first.items[0].IsDeleted || first.items[0].ExternalID != "p2" { + t.Fatalf("first run items = %#v", first.items) + } + if len(first.checkpoints) != 1 { + t.Fatalf("first run checkpoints = %d; want 1", len(first.checkpoints)) + } + second := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), first.checkpoints[0], second) + if err != nil { + t.Fatal(err) + } + if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" { + t.Fatalf("resume must tombstone only the remaining page: %#v", second.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p3"] != "" { + t.Fatalf("resume cursor = %#v", decoded) + } +} + +func TestFetchFullStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"}) + // p1 re-fetch succeeds and the p2 tombstone is checkpointed before the + // worker dies on the p3 tombstone. + first := &captureHandler{succeedFirst: 2} + if _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first); err == nil { + t.Fatal("interrupted full-sync tombstone run unexpectedly succeeded") + } + if len(first.items) != 2 || first.items[1].ExternalID != "p2" || !first.items[1].IsDeleted { + t.Fatalf("first run items = %#v", first.items) + } + if len(first.checkpoints) != 2 { + t.Fatalf("first run checkpoints = %d; want 2", len(first.checkpoints)) + } + deletionCheckpoint := decodeCursor(first.checkpoints[1]) + if !deletionCheckpoint.FullSync { + t.Fatalf("mid-full-sync checkpoint lost the resume flag: %#v", deletionCheckpoint) + } + if baseline := deletionCheckpoint.FullSyncBaseline["1"]; baseline["p1"] != "v:1" || baseline["p2"] != "" || + baseline["p3"] != "v:1" { + t.Fatalf("tombstone checkpoint did not advance the deletion baseline: %#v", deletionCheckpoint) + } + + resumeAPI := &streamAPI{pages: api.pages} + second := &captureHandler{} + next, err := newStreamConnector( + resumeAPI, + ).FetchFullStream(context.Background(), streamConfig(), first.checkpoints[1], second) + if err != nil { + t.Fatal(err) + } + if resumeAPI.bodyCalls != 0 { + t.Fatalf("resume re-fetched %d already synced page bodies", resumeAPI.bodyCalls) + } + if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" { + t.Fatalf("resume must tombstone only the remaining page: %#v", second.items) + } + decoded := decodeCursor(next) + if decoded.FullSync || decoded.FullSyncBaseline != nil { + t.Fatalf("completed full sync left resume fields set: %#v", decoded) + } + if decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "" || + decoded.SpacePages["1"]["p3"] != "" { + t.Fatalf("completed resume cursor = %#v", decoded) + } } func TestFetchStreamRefusesMassDeletion(t *testing.T) { @@ -275,6 +458,67 @@ func TestFetchStreamRefusesMassDeletion(t *testing.T) { } } +func TestFetchStreamRefusesMassDeletionInSurvivingScope(t *testing.T) { + prior := make(map[string]string, 100) + for i := 0; i < 100; i++ { + prior[fmt.Sprintf("p%03d", i)] = "v:1" + } + // The selection is unchanged; a broken listing suddenly returns only five + // of the hundred previously synced pages. + listed := make([]streamPage, 0, 5) + for i := 0; i < 5; i++ { + listed = append(listed, streamPage{id: fmt.Sprintf("p%03d", i), title: "Page", version: 1}) + } + api := &streamAPI{pages: listed} + h := &captureHandler{} + _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h) + if err == nil || len(h.items) != 0 { + t.Fatalf("surviving-scope mass deletion err=%v items=%#v", err, h.items) + } +} + +func TestFetchStreamAllowsExplicitSpaceToSubtreeShrink(t *testing.T) { + prior := make(map[string]string, 100) + for i := 0; i < 100; i++ { + prior[fmt.Sprintf("p%03d", i)] = "v:1" + } + // The user deliberately narrows a whole-space selection down to a page + // subtree. The mass-deletion guard must only protect a surviving root. + api := &streamAPI{pages: []streamPage{{id: "erp", title: "ERP", version: 2}}} + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:erp"} + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, streamCursor(prior), h) + if err != nil { + t.Fatalf("explicit scope shrink: %v", err) + } + if len(h.items) != len(prior)+1 || h.items[0].ExternalID != "erp" { + t.Fatalf("scope shrink items = %d, want selected page plus %d tombstones", len(h.items), len(prior)) + } + deleted := 0 + for _, item := range h.items[1:] { + if !item.IsDeleted { + t.Fatalf("scope shrink emitted a non-tombstone outside the new scope: %#v", item) + } + deleted++ + } + if deleted != len(prior) { + t.Fatalf("scope shrink tombstones = %d, want %d", deleted, len(prior)) + } + decoded := decodeCursor(next) + if len(decoded.SpacePages) != 1 || decoded.SpacePages["page:1:erp"]["erp"] != "v:2" { + t.Fatalf("scope shrink cursor retained cancelled roots: %#v", decoded.SpacePages) + } + if _, oldRootPresent := decoded.SpacePages["1"]; oldRootPresent { + t.Fatalf("cancelled space root returned to cursor: %#v", decoded.SpacePages) + } + for _, item := range h.items[1:] { + if item.SourceResourceID != "1" { + t.Fatalf("tombstone source = %q, want old root 1", item.SourceResourceID) + } + } +} + func TestFetchStreamRefusesEmptyListingWhenBaselineExists(t *testing.T) { prior := map[string]string{"p1": "v:1", "p2": "v:1"} api := &streamAPI{} @@ -349,6 +593,50 @@ func TestFetchFullStreamResumesFromCheckpoint(t *testing.T) { } } +func TestFetchFullStreamRetryRefetchesFailedPage(t *testing.T) { + pages := []streamPage{ + {id: "p1", title: "One", version: 1}, + {id: "p2", title: "Broken then healthy", version: 2}, + {id: "p3", title: "Three", version: 1}, + } + api := &streamAPI{pages: pages, bodyStatus: map[string]int{"p2": http.StatusForbidden}} + first := &captureHandler{checkpointErrAt: 2} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"}) + _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first) + if err == nil { + t.Fatal("interrupted full sync unexpectedly succeeded") + } + if len(first.checkpoints) != 1 { + t.Fatalf("first full-sync checkpoints = %d; want one successful checkpoint", len(first.checkpoints)) + } + checkpoint := decodeCursor(first.checkpoints[0]) + if !checkpoint.FullSync || checkpoint.FullSyncBaseline["1"]["p2"] != "v:1" || + checkpoint.SpacePages["1"]["p2"] != "" { + t.Fatalf("failed page was not left pending in the full-sync checkpoint: %#v", checkpoint) + } + + resumeAPI := &streamAPI{pages: pages} + second := &captureHandler{} + next, err := newStreamConnector(resumeAPI).FetchFullStream( + context.Background(), streamConfig(), first.checkpoints[0], second, + ) + if err != nil { + t.Fatal(err) + } + if resumeAPI.bodyCalls != 2 { + t.Fatalf("retry fetched %d bodies, want failed p2 and uncheckpointed p3", resumeAPI.bodyCalls) + } + if len(second.items) != 2 || second.items[0].ExternalID != "p2" || second.items[1].ExternalID != "p3" { + t.Fatalf("retry items = %#v; failed page was skipped", second.items) + } + decoded := decodeCursor(next) + if decoded.FullSync || decoded.FullSyncBaseline != nil || len(decoded.SpacePages) != 1 || + decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "v:2" || + decoded.SpacePages["1"]["p3"] != "v:1" { + t.Fatalf("completed full-sync cursor = %#v", decoded) + } +} + func TestFetchStreamContinuesAfterPageBodyError(t *testing.T) { api := &streamAPI{ pages: []streamPage{ @@ -430,57 +718,536 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) { } } -func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) { - for _, full := range []bool{false, true} { - t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) { - api := &streamAPI{ - endlessNext: true, - pages: []streamPage{{id: "p1", title: "Page", version: 2}}, - probe: map[string]probeReply{ - "gone": {status: http.StatusNotFound}, - "moved": {status: http.StatusOK, spaceKey: "OPS"}, - "kept": {status: http.StatusOK, spaceKey: "ENG"}, - "denied": {status: http.StatusForbidden}, - }, - } - h := &captureHandler{} - old := streamCursor(map[string]string{ - "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1", - }) - connector := newStreamConnector(api) - fetch := connector.FetchStream - if full { - fetch = connector.FetchFullStream - } - got, err := fetch(context.Background(), streamConfig(), old, h) - if err != nil { - t.Fatalf("fetch: %v", err) - } - if api.listCalls != 1+maxEmptyServerPages { - t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages) - } - if api.probeCalls != 4 { - t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls) - } - deleted := map[string]bool{} - for _, item := range h.items { - if item.IsDeleted { - deleted[item.ExternalID] = true - } - } - if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] { - t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted) - } - pages := decodeCursor(got).SpacePages["1"] - want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"} - if len(pages) != len(want) { - t.Fatalf("cursor pages = %#v, want %#v", pages, want) - } - for id, version := range want { - if pages[id] != version { - t.Fatalf("cursor pages = %#v, want %#v", pages, want) - } - } - }) +func TestListResourcesLoadsConfluencePagesLazily(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Home", version: 1}}} + connector := newStreamConnector(api) + spaces, err := connector.ListResources(context.Background(), streamConfig(), "") + if err != nil || len(spaces) != 1 || !spaces[0].HasChildren { + t.Fatalf("spaces = %#v, %v", spaces, err) + } + pages, err := connector.ListResources(context.Background(), streamConfig(), "1") + if err != nil || len(pages) != 1 { + t.Fatalf("top pages = %#v, %v", pages, err) + } + if pages[0].ExternalID != "page:1:p1" || pages[0].ParentID != "1" || pages[0].Type != "page" || + !pages[0].HasChildren { + t.Fatalf("page resource = %#v", pages[0]) + } + children, err := connector.ListResources(context.Background(), streamConfig(), "page:1:p1") + if err != nil || len(children) != 0 { + t.Fatalf("children = %#v, %v", children, err) + } + if api.listCalls != 1 { + t.Fatalf("space expansion made %d whole-space list calls", api.listCalls) + } + if api.ancestorCalls != 0 { + t.Fatalf("page expansion made %d unnecessary ancestor calls", api.ancestorCalls) + } +} + +func TestPickerSpaceCacheIsPartitionedByCredentialFingerprint(t *testing.T) { + api := &streamAPI{} + connector := newStreamConnector(api) + client, cfg, err := connector.configured(streamConfig()) + if err != nil { + t.Fatal(err) + } + if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil { + t.Fatal(err) + } + if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil { + t.Fatal(err) + } + rotated := cfg + rotated.secret = "rotated-password" + if _, err := connector.pickerSpaces(context.Background(), client, rotated); err != nil { + t.Fatal(err) + } + if api.spaceCalls != 2 { + t.Fatalf("space list calls = %d; want 2 for distinct credentials", api.spaceCalls) + } +} + +func TestPickerSpaceCachePrunesExpiredCredentialEntries(t *testing.T) { + api := &streamAPI{} + connector := newStreamConnector(api) + client, cfg, err := connector.configured(streamConfig()) + if err != nil { + t.Fatal(err) + } + expired := cfg + expired.secret = "expired-password" + connector.spaceCache.entries = map[string]cachedSpaces{ + pickerSpaceCacheKey(expired): {expires: time.Now().Add(-time.Second)}, + } + if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil { + t.Fatal(err) + } + if len(connector.spaceCache.entries) != 1 { + t.Fatalf("cache entries = %#v; expired credential entry was retained", connector.spaceCache.entries) + } +} + +func TestFetchStreamSyncsSelectedPageSubtree(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}} + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 1 || h.items[0].SourceResourceID != "page:1:p1" || h.items[0].ExternalID != "p1" { + t.Fatalf("items = %#v", h.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:2" { + t.Fatalf("cursor = %#v", decoded) + } +} + +func TestFetchStreamReconcilesCancelledSpaceScope(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}} + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "p2" { + t.Fatalf("items = %#v", h.items) + } + decoded := decodeCursor(next) + if _, oldScopePresent := decoded.SpacePages["1"]; oldScopePresent || + decoded.SpacePages["page:1:p1"]["p1"] != "v:2" { + t.Fatalf("cursor = %#v", decoded) + } +} + +func TestFetchStreamPreservesMovedPageWhenBodyFails(t *testing.T) { + api := &streamAPI{ + pages: []streamPage{{id: "p1", title: "Selected", version: 2}}, + bodyStatus: map[string]int{"p1": http.StatusNotFound}, + } + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h) + if err != nil { + t.Fatal(err) + } + decoded := decodeCursor(next) + if decoded.SpacePages["page:1:p1"]["p1"] != "v:1" { + t.Fatalf("prior version was not preserved: %#v", decoded) + } + deletedP2 := false + for _, item := range h.items { + if item.IsDeleted && item.ExternalID == "p1" { + t.Fatalf("failed page was deleted: %#v", h.items) + } + deletedP2 = deletedP2 || (item.IsDeleted && item.ExternalID == "p2") + } + if !deletedP2 { + t.Fatalf("cancelled scope page was not deleted: %#v", h.items) + } +} + +func TestFetchStreamDoesNotDeleteWhenNewScopeCannotEnumerate(t *testing.T) { + api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}} + connector := newStreamConnector(api) + originalFactory := connector.newClient + connector.newClient = func(cfg config) (*client, error) { + c, err := originalFactory(cfg) + if err != nil { + return nil, err + } + originalTransport := c.http.Transport + c.http.Transport = roundTripper(func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/wiki/rest/api/content/p1" && req.URL.Query().Get("expand") == "version,space" { + return &http.Response{ + StatusCode: http.StatusNotFound, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("missing")), + }, nil + } + return originalTransport.RoundTrip(req) + }) + return c, nil + } + ds := streamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + h := &captureHandler{} + _, err := connector.FetchStream( + context.Background(), + ds, + streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}), + h, + ) + if err == nil { + t.Fatal("scope enumeration failure unexpectedly succeeded") + } + for _, item := range h.items { + if item.IsDeleted { + t.Fatalf("incomplete scope emitted a tombstone: %#v", h.items) + } + } +} + +func TestBuildSyncPlanNormalizesOverlappingScopes(t *testing.T) { + api := &streamAPI{} + client := &client{ + cfg: config{baseURL: "https://confluence.test/wiki"}, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/rest/api/space": + return api.jsonResponse( + map[string]any{"results": []any{map[string]any{"id": 1, "key": "ENG", "name": "Engineering"}}}, + ) + case "/wiki/rest/api/content/parent": + return api.jsonResponse( + map[string]any{"id": "parent", "space": map[string]any{"key": "ENG"}, "ancestors": []any{}}, + ) + case "/wiki/rest/api/content/child": + return api.jsonResponse( + map[string]any{ + "id": "child", + "space": map[string]any{"key": "ENG"}, + "ancestors": []any{map[string]any{"id": "parent"}}, + }, + ) + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + } + connector := NewConnector() + plan, err := connector.buildSyncPlan(context.Background(), client, []string{"page:1:child", "page:1:parent"}) + if err != nil { + t.Fatal(err) + } + if len(plan.Roots) != 1 || plan.Roots[0].ResourceID != "page:1:parent" { + t.Fatalf("page overlap plan = %#v", plan) + } + plan, err = connector.buildSyncPlan(context.Background(), client, []string{"1", "page:1:child"}) + if err != nil { + t.Fatal(err) + } + if len(plan.Roots) != 1 || plan.Roots[0].Kind != syncWholeSpace || plan.Roots[0].ResourceID != "1" { + t.Fatalf("space overlap plan = %#v", plan) + } +} + +func TestFetchStreamCloudSyncsSelectedPageSubtree(t *testing.T) { + api := &streamAPI{cloud: true, pages: []streamPage{{id: "p1", title: "Cloud", version: 3}}} + ds := cloudStreamConfig() + ds.ResourceIDs = []string{"page:1:p1"} + h := &captureHandler{} + next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 1 || h.items[0].ExternalID != "p1" || h.items[0].SourceResourceID != "page:1:p1" { + t.Fatalf("items = %#v", h.items) + } + if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:3" { + t.Fatalf("cursor = %#v", decoded) + } +} + +func TestListResourcesCloudStaysLazyWithContainerLimitation(t *testing.T) { + api := &streamAPI{} + var listDepths []string + conn := &Connector{newClient: func(cfg config) (*client, error) { + return &client{ + cfg: cfg, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/api/v2/spaces": + return api.jsonResponse( + map[string]any{ + "results": []any{map[string]any{"id": "1", "key": "ENG", "name": "Engineering"}}, + }, + ) + case "/wiki/api/v2/spaces/1/pages": + listDepths = append(listDepths, req.URL.Query().Get("depth")) + // depth=0 surfaces root pages only; the folder and the page nested + // inside it are unreachable through this endpoint (CONFCLOUD-84275). + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "top", "type": "page", "title": "Top"}, + map[string]any{"id": "folder", "type": "folder", "title": "Folder"}, + }}) + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + }, nil + }} + // Cloud spaces carry the limitation marker so the picker can explain why + // pages under top-level containers are not selectable. + spaces, err := conn.ListResources(context.Background(), cloudStreamConfig(), "") + if err != nil { + t.Fatal(err) + } + if len(spaces) != 1 || spaces[0].Metadata["hierarchy_limitation"] != "cloud_top_level_containers" { + t.Fatalf("cloud space metadata = %#v", spaces) + } + resources, err := conn.ListResources(context.Background(), cloudStreamConfig(), "1") + if err != nil { + t.Fatal(err) + } + if len(resources) != 1 || resources[0].ExternalID != "page:1:top" { + t.Fatalf("resources = %#v", resources) + } + // Expanding a space must stay a single shallow call, never a whole-space + // scan, even though top-level folders hide some pages. + if len(listDepths) != 1 || listDepths[0] != "0" { + t.Fatalf("space expansion depths = %#v; want a single depth=0 call", listDepths) + } +} + +func TestResolveResourceAncestorsSkipsUnresolvableSelections(t *testing.T) { + api := &streamAPI{} + connector := &Connector{newClient: func(cfg config) (*client, error) { + return &client{ + cfg: cfg, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/rest/api/space": + return api.jsonResponse(map[string]any{"results": []any{map[string]any{ + "id": 1, "key": "ENG", "name": "Engineering", + }}}) + case "/wiki/rest/api/content/alive": + return api.jsonResponse(map[string]any{ + "id": "alive", "space": map[string]any{"key": "ENG"}, + "ancestors": []any{map[string]any{"id": "parent"}}, + }) + case "/wiki/rest/api/content/gone": + return &http.Response{ + StatusCode: http.StatusNotFound, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("missing page")), + }, nil + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + }, nil + }} + ancestors, err := connector.ResolveResourceAncestors( + context.Background(), streamConfig(), + []string{"page:1:alive", "page:1:gone", "page:9:elsewhere", "not-a-resource"}, + ) + if err != nil { + t.Fatal(err) + } + want := []string{"1", "page:1:parent"} + if len(ancestors) != len(want) || ancestors[0] != want[0] || ancestors[1] != want[1] { + t.Fatalf("ancestors = %#v; want %#v", ancestors, want) + } +} + +func TestVisibleCloudPageChildrenTraversesContainers(t *testing.T) { + api := &streamAPI{} + client := &client{ + cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"}, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + switch req.URL.Path { + case "/wiki/api/v2/pages/a/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "b", "type": "page", "title": "B"}, + map[string]any{"id": "folder", "type": "folder", "title": "Folder"}, + }}) + case "/wiki/api/v2/folders/folder/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "c", "type": "page", "title": "C"}, + map[string]any{"id": "db", "type": "database", "title": "DB"}, + }}) + case "/wiki/api/v2/databases/db/direct-children": + return api.jsonResponse( + map[string]any{"results": []any{map[string]any{"id": "e", "type": "page", "title": "E"}}}, + ) + default: + return nil, errors.New("unexpected endpoint: " + req.URL.String()) + } + })}, + } + pages, err := client.visibleCloudPageChildren(context.Background(), "a") + if err != nil { + t.Fatal(err) + } + if len(pages) != 3 || pages[0].ID != "b" || pages[1].ID != "c" || pages[2].ID != "e" { + t.Fatalf("projected pages = %#v", pages) + } +} + +func TestCloudPageSubtreeWalksDeepMixedHierarchy(t *testing.T) { + api := &streamAPI{} + client := &client{ + cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"}, + http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + return nil, fmt.Errorf("unexpected method %s", req.Method) + } + switch req.URL.Path { + case "/wiki/api/v2/pages/root": + return api.jsonResponse(map[string]any{"id": "root", "spaceId": "1"}) + case "/wiki/api/v2/pages/p1": + return api.jsonResponse(map[string]any{ + "id": "p1", "spaceId": "1", "version": map[string]any{"number": 2}, + }) + case "/wiki/api/v2/pages/p2": + return api.jsonResponse(map[string]any{ + "id": "p2", "spaceId": "1", "version": map[string]any{"number": 3}, + }) + case "/wiki/api/v2/pages/p3": + return api.jsonResponse(map[string]any{ + "id": "p3", "spaceId": "1", "version": map[string]any{"number": 4}, + }) + case "/wiki/api/v2/pages/root/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "p1", "type": "page", "title": "First page"}, + map[string]any{"id": "folder", "type": "folder", "title": "Folder"}, + }}) + case "/wiki/api/v2/pages/p1/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "db", "type": "database", "title": "Database"}, + }}) + case "/wiki/api/v2/folders/folder/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "p2", "type": "page", "title": "Second page"}, + }}) + case "/wiki/api/v2/databases/db/direct-children": + return api.jsonResponse(map[string]any{"results": []any{ + map[string]any{"id": "p3", "type": "page", "title": "Deep page"}, + }}) + case "/wiki/api/v2/pages/p2/direct-children", "/wiki/api/v2/pages/p3/direct-children": + return api.jsonResponse(map[string]any{"results": []any{}}) + default: + return nil, fmt.Errorf("unexpected endpoint: %s", req.URL.String()) + } + })}, + } + + pages, err := client.pageSubtree(context.Background(), space{ID: "1"}, "root") + if err != nil { + t.Fatal(err) + } + want := []string{"root", "p1", "p2", "p3"} + if len(pages) != len(want) { + t.Fatalf("subtree pages = %#v; want IDs %v", pages, want) + } + for i, id := range want { + if pages[i].ID != id || pages[i].SpaceID != "1" { + t.Fatalf("subtree page[%d] = %#v; want ID %q in space 1", i, pages[i], id) + } + } +} + +func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) { + api := &streamAPI{ + endlessNext: true, + pages: []streamPage{{id: "p1", title: "Page", version: 2}}, + probe: map[string]probeReply{ + "gone": {status: http.StatusNotFound}, + "kept": {status: http.StatusOK, spaceKey: "ENG"}, + }, + } + h := &captureHandler{} + old := streamCursor(map[string]string{"p1": "v:1", "gone": "v:1", "kept": "v:1"}) + got, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if api.listCalls != 1+maxEmptyServerPages || api.probeCalls != 2 { + t.Fatalf("list/probe calls = %d/%d", api.listCalls, api.probeCalls) + } + if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "gone" { + t.Fatalf("items = %#v", h.items) + } + pages := decodeCursor(got).SpacePages["1"] + if pages["p1"] != "v:2" || pages["kept"] != "v:1" || len(pages) != 2 { + t.Fatalf("cursor pages = %#v", pages) + } +} + +func TestFetchStreamDoesNotRestoreCancelledRootForPageFoundByPartialProbe(t *testing.T) { + api := &streamAPI{ + endlessNext: true, + probe: map[string]probeReply{ + "kept": {status: http.StatusOK, spaceKey: "ENG"}, + }, + } + h := &captureHandler{} + old := (&cursor{SpacePages: map[string]map[string]string{ + "page:1:old-root": {"kept": "v:1"}, + }}).syncCursor() + next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err != nil { + t.Fatal(err) + } + if len(h.items) != 0 { + t.Fatalf("confirmed-present page emitted items: %#v", h.items) + } + decoded := decodeCursor(next) + if len(decoded.SpacePages) != 1 || decoded.SpacePages["1"]["kept"] != "v:1" { + t.Fatalf("reconciled cursor = %#v; expected only the current root", decoded.SpacePages) + } + if _, oldRootPresent := decoded.SpacePages["page:1:old-root"]; oldRootPresent { + t.Fatalf("cancelled root was restored to cursor: %#v", decoded.SpacePages) + } +} + +func TestFetchStreamDoesNotTombstoneWhenPartialListingProbeFails(t *testing.T) { + api := &streamAPI{ + endlessNext: true, + probe: map[string]probeReply{ + "gone": {status: http.StatusNotFound}, + "unknown": {status: http.StatusForbidden}, + }, + } + h := &captureHandler{} + old := streamCursor(map[string]string{"gone": "v:1", "unknown": "v:1"}) + _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h) + if err == nil { + t.Fatal("unverified partial-listing candidate unexpectedly completed reconciliation") + } + for _, item := range h.items { + if item.IsDeleted { + t.Fatalf("probe failure allowed a tombstone: %#v", h.items) + } + } + if len(h.checkpoints) != 0 { + t.Fatalf("probe failure checkpointed reconciliation: %#v", h.checkpoints) + } + if api.probeCalls == 0 { + t.Fatal("empty incomplete listing was rejected before checking prior pages") + } +} + +func TestFetchStreamDefersWhenPartialListingProbeBudgetIsExhausted(t *testing.T) { + prior := make(map[string]string, maxDeletionProbes+1) + probes := make(map[string]probeReply, maxDeletionProbes+1) + for i := 0; i <= maxDeletionProbes; i++ { + id := fmt.Sprintf("p%03d", i) + prior[id] = "v:1" + probes[id] = probeReply{status: http.StatusNotFound} + } + api := &streamAPI{endlessNext: true, probe: probes} + h := &captureHandler{} + _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h) + if err == nil { + t.Fatal("partial listing with unverified candidates unexpectedly completed") + } + if api.probeCalls != maxDeletionProbes { + t.Fatalf("existence probes = %d; want capped at %d", api.probeCalls, maxDeletionProbes) + } + if len(h.items) != 0 || len(h.checkpoints) != 0 { + t.Fatalf( + "probe budget exhaustion emitted or checkpointed reconciliation: items=%#v checkpoints=%#v", + h.items, + h.checkpoints, + ) } } diff --git a/internal/datasource/connector/confluence/resource.go b/internal/datasource/connector/confluence/resource.go new file mode 100644 index 00000000..a34a1619 --- /dev/null +++ b/internal/datasource/connector/confluence/resource.go @@ -0,0 +1,43 @@ +package confluence + +import ( + "fmt" + "strings" +) + +type resourceKind uint8 + +const ( + resourceSpace resourceKind = iota + resourcePage +) + +// resourceRef is the canonical, persisted representation of a selection. +// Space IDs deliberately remain unprefixed for backwards compatibility. +type resourceRef struct { + Kind resourceKind + SpaceID string + PageID string +} + +func makePageResourceID(spaceID, pageID string) string { + return "page:" + spaceID + ":" + pageID +} + +func parseResourceID(id string) (resourceRef, error) { + id = strings.TrimSpace(id) + if id == "" { + return resourceRef{}, fmt.Errorf("invalid empty Confluence resource ID") + } + if !strings.HasPrefix(id, "page:") { + if strings.Contains(id, ":") { + return resourceRef{}, fmt.Errorf("invalid Confluence space resource ID %q", id) + } + return resourceRef{Kind: resourceSpace, SpaceID: id}, nil + } + parts := strings.Split(id, ":") + if len(parts) != 3 || parts[0] != "page" || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" { + return resourceRef{}, fmt.Errorf("invalid Confluence page resource ID %q", id) + } + return resourceRef{Kind: resourcePage, SpaceID: parts[1], PageID: parts[2]}, nil +} diff --git a/internal/datasource/connector/confluence/resource_test.go b/internal/datasource/connector/confluence/resource_test.go new file mode 100644 index 00000000..b03e92f2 --- /dev/null +++ b/internal/datasource/connector/confluence/resource_test.go @@ -0,0 +1,23 @@ +package confluence + +import "testing" + +func TestParseResourceID(t *testing.T) { + space, err := parseResourceID("86802433") + if err != nil || space.Kind != resourceSpace || space.SpaceID != "86802433" { + t.Fatalf("space = %#v, %v", space, err) + } + page, err := parseResourceID("page:86802433:123456789") + if err != nil || page.Kind != resourcePage || page.SpaceID != "86802433" || page.PageID != "123456789" { + t.Fatalf("page = %#v, %v", page, err) + } + invalid := []string{ + "", "page:", "page:100", "page::200", "page:100:", "page:100:200:300", + "node:86802433:folder-1", "space:100", + } + for _, id := range invalid { + if _, err := parseResourceID(id); err == nil { + t.Errorf("parseResourceID(%q) succeeded", id) + } + } +} diff --git a/internal/datasource/connector/confluence/sync_plan.go b/internal/datasource/connector/confluence/sync_plan.go new file mode 100644 index 00000000..c61484d9 --- /dev/null +++ b/internal/datasource/connector/confluence/sync_plan.go @@ -0,0 +1,112 @@ +package confluence + +import ( + "context" + "fmt" + "sort" +) + +type syncRootKind uint8 + +const ( + syncWholeSpace syncRootKind = iota + syncPageSubtree +) + +type syncRoot struct { + ResourceID string + SpaceID string + Space space + PageID string + Kind syncRootKind +} + +type syncPlan struct{ Roots []syncRoot } + +// buildSyncPlan validates selections before any destructive reconciliation and +// removes scopes which are already covered by a selected ancestor. +func (c *Connector) buildSyncPlan(ctx context.Context, client *client, resourceIDs []string) (*syncPlan, error) { + spaces, err := client.spaces(ctx) + if err != nil { + return nil, fmt.Errorf("list Confluence spaces: %w", err) + } + byID := make(map[string]space, len(spaces)) + for _, s := range spaces { + byID[s.ID] = s + } + + plan := &syncPlan{} + wholeSpaces := make(map[string]bool) + pageRoots := make(map[string]syncRoot) + ancestorPaths := make(map[string][]page) + for _, resourceID := range resourceIDs { + ref, err := parseResourceID(resourceID) + if err != nil { + return nil, err + } + s, ok := byID[ref.SpaceID] + if !ok { + return nil, fmt.Errorf( + "selected Confluence space %s is unavailable; refusing destructive reconciliation", + ref.SpaceID, + ) + } + if ref.Kind == resourceSpace { + wholeSpaces[ref.SpaceID] = true + continue + } + // pageAncestors fetches the page itself on Server/DC and Cloud; that is + // also the authoritative check that a client did not forge its space ID. + ancestors, err := client.pageAncestors(ctx, s, ref.PageID) + if err != nil { + return nil, fmt.Errorf("validate selected Confluence page %s: %w", ref.PageID, err) + } + pageRoots[resourceID] = syncRoot{ + ResourceID: resourceID, + SpaceID: ref.SpaceID, + Space: s, + PageID: ref.PageID, + Kind: syncPageSubtree, + } + ancestorPaths[resourceID] = ancestors + for _, ancestor := range ancestors { + if ancestor.ID == ref.PageID { + return nil, fmt.Errorf("invalid cyclic Confluence page ancestry for %s", ref.PageID) + } + } + } + spaceIDs := make([]string, 0, len(wholeSpaces)) + for id := range wholeSpaces { + spaceIDs = append(spaceIDs, id) + } + sort.Strings(spaceIDs) + for _, id := range spaceIDs { + plan.Roots = append(plan.Roots, syncRoot{ResourceID: id, SpaceID: id, Space: byID[id], Kind: syncWholeSpace}) + } + pageIDs := make([]string, 0, len(pageRoots)) + for id := range pageRoots { + pageIDs = append(pageIDs, id) + } + sort.Strings(pageIDs) + for _, id := range pageIDs { + root := pageRoots[id] + if wholeSpaces[root.SpaceID] { + continue + } + ancestors := ancestorPaths[id] + covered := false + for _, ancestor := range ancestors { + if ancestor.Kind != "" && ancestor.Kind != "page" { + continue + } + if _, ok := pageRoots[makePageResourceID(root.SpaceID, ancestor.ID)]; ok { + covered = true + break + } + } + if !covered { + plan.Roots = append(plan.Roots, pageRoots[id]) + } + } + return plan, nil +} diff --git a/internal/datasource/connector/confluence/types.go b/internal/datasource/connector/confluence/types.go index 7884f349..49828808 100644 --- a/internal/datasource/connector/confluence/types.go +++ b/internal/datasource/connector/confluence/types.go @@ -95,10 +95,11 @@ func isAtlassianCloudHost(host string) bool { } type space struct { - ID string `json:"id"` - Key string `json:"key"` - Name string `json:"name"` - Links struct { + ID string `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + HomepageID string `json:"homepageId"` + Links struct { WebUI string `json:"webui"` } `json:"_links"` } @@ -125,10 +126,13 @@ type serverSpaceList struct { } type page struct { - ID string `json:"id"` - Title string `json:"title"` - Status string `json:"status"` - Space struct { + ID string `json:"id"` + Title string `json:"title"` + Status string `json:"status"` + Kind string `json:"-"` + SpaceID string `json:"spaceId"` + Ancestors []page `json:"ancestors"` + Space struct { Key string `json:"key"` Name string `json:"name"` } `json:"space"` @@ -189,6 +193,27 @@ type cloudPage struct { } `json:"_links"` } +// cloudHierarchyItem is shared by Cloud direct-children, descendants, and +// ancestors. Non-page items are intentionally retained by the API model so we +// can filter them explicitly at the navigation/import boundary. +type cloudHierarchyItem struct { + ID string `json:"id"` + Type string `json:"type"` + Title string `json:"title"` + SpaceID string `json:"spaceId"` + Status string `json:"status"` + Links struct { + WebUI string `json:"webui"` + } `json:"_links"` +} + +type cloudHierarchyList struct { + Results []cloudHierarchyItem `json:"results"` + Links struct { + Next string `json:"next"` + } `json:"_links"` +} + type cloudPageList struct { Results []cloudPage `json:"results"` Links struct { @@ -278,14 +303,37 @@ func prepareSyncCursors(old *types.SyncCursor, forceFull bool) (baseline, next c return baseline, next } -func pageVersion(p page) string { +// removePageOwnership drops a page from every root ownership in the cursor. +// It runs after each tombstone emit so the checkpoint mirrors the deletion; +// otherwise the page would linger as a ghost, be re-tombstoned on every later +// run, and eventually trip the mass-deletion guard. During a full sync the +// deletion baseline lives in FullSyncBaseline, not SpacePages (which is being +// rebuilt from scratch), so both maps must record the completed tombstone for +// a retry to resume with the remaining candidates only. +func removePageOwnership(c *cursor, pageID string) { + removeFromPageMap(c.SpacePages, pageID) + if c.FullSync { + removeFromPageMap(c.FullSyncBaseline, pageID) + } +} + +func removeFromPageMap(m map[string]map[string]string, pageID string) { + for _, pages := range m { + delete(pages, pageID) + } +} + +func pageVersion(p page) (string, bool) { if p.Version.Number > 0 { - return fmt.Sprintf("v:%d", p.Version.Number) + return fmt.Sprintf("v:%d", p.Version.Number), true } if p.Version.When != "" { - return "t:" + p.Version.When + return "t:" + p.Version.When, true } - return "t:" + p.Version.CreatedAt + if p.Version.CreatedAt != "" { + return "t:" + p.Version.CreatedAt, true + } + return "", false } func pageUpdatedAt(p page) time.Time { diff --git a/internal/datasource/connector/confluence/types_test.go b/internal/datasource/connector/confluence/types_test.go index 6a2247f0..bf62f7f3 100644 --- a/internal/datasource/connector/confluence/types_test.go +++ b/internal/datasource/connector/confluence/types_test.go @@ -75,6 +75,34 @@ func TestPageFileNameIncludesID(t *testing.T) { } } +func TestPageVersionUnknown(t *testing.T) { + withNumber := page{} + withNumber.Version.Number = 7 + withWhen := page{} + withWhen.Version.When = "2026-01-02T03:04:05Z" + withCreatedAt := page{} + withCreatedAt.Version.CreatedAt = "2026-01-02T03:04:05Z" + + for _, tc := range []struct { + name string + in page + want string + wantKnown bool + }{ + {"number", withNumber, "v:7", true}, + {"when", withWhen, "t:2026-01-02T03:04:05Z", true}, + {"createdAt", withCreatedAt, "t:2026-01-02T03:04:05Z", true}, + {"unknown", page{}, "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, known := pageVersion(tc.in) + if got != tc.want || known != tc.wantKnown { + t.Fatalf("pageVersion() = (%q, %v); want (%q, %v)", got, known, tc.want, tc.wantKnown) + } + }) + } +} + func TestPrepareSyncCursorsPreservesFullSyncBaseline(t *testing.T) { old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) baseline, next := prepareSyncCursors(old, true)