feat(confluence): add directory sync and private image assets (#3319)

* feat(confluence): support page subtree selection

* feat(confluence): add private image asset resolver

* fix(confluence): preserve pages during partial syncs

* fix(confluence): cap total inlined image bytes per page

Up to 30 images of 10 MB each could be inlined as base64, and every
downstream conversion copies the page again, so one extreme page could
peak above 1 GB. Cap the data URIs a page holds, in the download cache
and the rewritten HTML, at 50 MB; images past the budget keep their
original src like any other skipped image.

---------

Co-authored-by: wizardchen <wizardchen@tencent.com>
Co-authored-by: lyingbug <xiayula2014@gmail.com>
main
jacentsao 2026-09-25 22:23:48 +08:00 committed by GitHub
parent a7fc9e13da
commit e7c27ccc9e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 3031 additions and 251 deletions

View File

@ -82,6 +82,8 @@ export interface Resource {
url: string
parent_id?: string
has_children?: boolean
/** Connector-specific hints, e.g. Confluence Cloud's top-level-container limitation. */
metadata?: Record<string, any>
}
// --- API calls ---

View File

@ -6979,6 +6979,9 @@ export default {
authHeaders: 'Custom headers (optional)',
authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx'
},
confluence: {
cloudFolderLimitation: 'Confluence Cloud cannot list pages stored directly under top-level folders in this space; selecting the whole space still syncs them.'
},
comingSoon: 'Coming soon',
docHint: 'Get credentials at:',
openDoc: 'Open documentation',

View File

@ -6979,6 +6979,9 @@ export default {
authHeaders: 'カスタムヘッダー(任意)',
authHeadersHint: '非公開フィード用です。1行に1つ「名前: 値」の形式で入力します(例: Authorization: Bearer xxxx)'
},
confluence: {
cloudFolderLimitation: 'Confluence Cloud では、スペース直下のフォルダーなどコンテナ配下のページをここに一覧できません。スペース全体を選択すれば同期されます。'
},
comingSoon: '近日対応予定',
docHint: '認証情報の取得先:',
openDoc: 'ドキュメントを開く',

View File

@ -1125,6 +1125,9 @@ export default {
authHeaders: '사용자 지정 헤더 (선택)',
authHeadersHint: '비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx'
},
confluence: {
cloudFolderLimitation: 'Confluence Cloud는 아직 스페이스 최상위 폴더 등 컨테이너 아래의 페이지를 여기에 나열할 수 없습니다. 스페이스 전체를 선택하면 함께 동기화됩니다.'
},
connectorDesc: {
feishu: '페이슈 위키에서 문서, 스프레드시트, 파일 동기화',
lark: 'Lark 위키에서 문서, 스프레드시트, 파일 동기화',

View File

@ -1125,6 +1125,9 @@ export default {
authHeaders: 'Пользовательские заголовки (необязательно)',
authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx'
},
confluence: {
cloudFolderLimitation: 'Confluence Cloud пока не может показать здесь страницы из папок верхнего уровня пространства; при выборе всего пространства они будут синхронизированы.'
},
connectorDesc: {
feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki',
lark: 'Синхронизация документов, таблиц и файлов из Lark Wiki',

View File

@ -1127,6 +1127,9 @@ export default {
authHeaders: '自定义请求头(可选)',
authHeadersHint: '用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx'
},
confluence: {
cloudFolderLimitation: 'Confluence Cloud 暂不支持在此列出空间顶层文件夹等容器下的页面;选择整个空间仍会同步它们。'
},
connectorDesc: {
feishu: '同步飞书知识库中的文档、表格、文件',
lark: '同步 Lark 知识库中的文档、表格、文件(飞书国际版)',

View File

@ -157,3 +157,25 @@ test('new GitLab data sources continue to test credentials without persistence',
assert.deepEqual(f.calls.map(call => call.method), ['validateCredentials'])
} finally { f.close() }
})
test('Cloud hierarchy limitation stays visible after an empty space expansion', async () => {
const f = await fixture()
try {
f.vm.resources = [{
external_id: 'space-1',
name: 'Cloud space',
type: 'space',
has_children: true,
metadata: { hierarchy_limitation: 'cloud_top_level_containers' },
}]
f.vm.expandedResourceIds = new Set(['space-1'])
await nextTick()
assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true)
f.vm.expandedResourceIds = new Set()
f.vm.loadedChildrenIds = new Set(['space-1'])
f.vm.resources = [{ ...f.vm.resources[0], has_children: false }]
await nextTick()
assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true)
} finally { f.close() }
})

View File

@ -449,6 +449,16 @@ async function ensureChildrenLoaded(id: string) {
if (!existing.has(c.external_id)) merged.push(c)
}
resources.value = merged
} else {
// Connectors may conservatively advertise HasChildren to avoid an N+1
// probe. Once lazy loading proves this is a leaf, collapse it and retain
// that fact so another expand cannot trigger another empty request.
resources.value = resources.value.map(r => r.external_id === id
? { ...r, has_children: false }
: r)
const expanded = new Set(expandedResourceIds.value)
expanded.delete(id)
expandedResourceIds.value = expanded
}
loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)
} catch (e: any) {
@ -466,13 +476,21 @@ async function ensureChildrenLoaded(id: string) {
const visibleTree = computed(() => {
const roots = resources.value.filter(r => !r.parent_id)
const result: { resource: Resource; depth: number }[] = []
const result: { resource: Resource; depth: number; noticeAfter?: boolean }[] = []
function walk(items: Resource[], depth: number) {
for (const r of items) {
result.push({ resource: r, depth })
if (r.has_children && expandedResourceIds.value.has(r.external_id)) {
walk(childrenMap.value.get(r.external_id) || [], depth + 1)
}
// Keep this visible after an empty root listing is recognized as a leaf.
if (
r.metadata?.hierarchy_limitation === 'cloud_top_level_containers' &&
(expandedResourceIds.value.has(r.external_id) ||
(!r.has_children && loadedChildrenIds.value.has(r.external_id)))
) {
result.push({ resource: r, depth: depth + 1, noticeAfter: true })
}
}
}
walk(roots, 0)
@ -1720,9 +1738,12 @@ const drawerConfirmText = computed(() => {
</div>
</div>
<div class="resource-picker__list" role="tree">
<template
v-for="{ resource: r, depth, noticeAfter } in visibleTree"
:key="noticeAfter ? `${r.external_id}__notice` : r.external_id"
>
<div
v-for="{ resource: r, depth } in visibleTree"
:key="r.external_id"
v-if="!noticeAfter"
class="resource-picker__row"
:class="{
'is-checked': resourceRowState(r.external_id) === 'checked',
@ -1787,6 +1808,14 @@ const drawerConfirmText = computed(() => {
>{{ resourceTypeLabel(r.type) }}</span>
</span>
</div>
<p
v-else
class="resource-picker__notice"
:style="{ '--depth': depth }"
>
{{ t('datasource.confluence.cloudFolderLimitation') }}
</p>
</template>
</div>
</div>
<div v-else class="ds-resource-empty">
@ -2576,6 +2605,19 @@ const drawerConfirmText = computed(() => {
background: color-mix(in srgb, var(--td-text-color-placeholder) 8%, transparent);
}
.resource-picker__notice {
--depth: 0;
margin: 0 0 2px;
padding: 4px 8px 4px calc(8px + var(--depth) * 14px);
font-size: var(--app-text-xs);
line-height: 1.5;
color: var(--td-text-color-placeholder);
}
.resource-picker__notice:last-child {
margin-bottom: 0;
}
/* --- Step 2: empty state --- */
.ds-resource-empty {
text-align: center;

View File

@ -0,0 +1,301 @@
package confluence
import (
"context"
"encoding/base64"
"errors"
"html"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/logger"
)
const (
// maxPageImages is the total data URI image budget one page may carry into the
// downstream docparser, which enforces a single per-page limit (maxRemoteImages)
// across every data:image URI regardless of origin. The budget is shared: it
// covers data URI images already present in the page plus the ones the resolver
// generates, and a repeated image consumes it per occurrence, not per unique URL.
// Downloads stay deduplicated by URL, and only occurrences within the remaining
// budget are collected, so the connector never emits more data URIs than the
// pipeline will keep.
maxPageImages = 30
// maxPageInlineBytes caps the total size of the data URIs one page may carry,
// both in the download cache and in the rewritten HTML. Without it an extreme
// page (maxPageImages occurrences of maxImageBytes images) would inline hundreds
// of MB of base64, copied again by every downstream conversion. Images past the
// budget keep their original src, like any other image the resolver skips.
maxPageInlineBytes = 50 << 20
// imageDownloadConcurrency bounds in-flight image downloads per page so an
// image-heavy page cannot exhaust HTTP connections or spike memory.
imageDownloadConcurrency = 4
)
// errImagePageBudget marks an image dropped because the page's inline budget
// (maxPageInlineBytes) was already used up.
var errImagePageBudget = errors.New("confluence page image budget exhausted")
// imgTagRe matches a full <img ...> start tag, tolerating '>' inside quoted
// attribute values. The \b keeps it from matching <image>.
var imgTagRe = regexp.MustCompile(`(?i)<img\b(?:[^>"']|"[^"]*"|'[^']*')*>`)
// srcAttrRe captures the src attribute value. The leading \s and the value group
// keep it from matching data-src/srcset, and support double-quoted,
// single-quoted, or bare values. Group 1 is the "src=" prefix, group 2 the value.
var srcAttrRe = regexp.MustCompile(`(?i)(\ssrc\s*=\s*)("[^"]*"|'[^']*'|[^\s"'>]+)`)
type imgAction uint8
const (
imgKeep imgAction = iota // empty, data:, external, or non-http scheme: leave untouched
imgDownload // same-origin private candidate
)
// assetResolver rewrites same-origin private Confluence images in a page's
// body.view HTML into inline data URIs. Persistence and resource:// rewriting
// are left entirely to the downstream docparser image pipeline; the resolver
// only performs the one step that pipeline cannot do on its own: an
// authenticated fetch of Confluence-private bytes.
type assetResolver struct {
client *client
}
func newAssetResolver(c *client) *assetResolver { return &assetResolver{client: c} }
// Resolve returns content with same-origin private <img src> occurrences replaced
// by data URIs, within the per-page budget that Resolve shares with any data URI
// images already in the page. It is best-effort: an image that is skipped, falls
// past the remaining budget, or fails to download keeps its original src, and image
// problems never fail the page. Downloads are deduplicated by URL, but the budget
// counts written data URIs (occurrences) to match the downstream pipeline, which
// caps every data:image URI per page regardless of where it came from.
func (r *assetResolver) Resolve(ctx context.Context, content string) string {
if r == nil || r.client == nil {
return content
}
// Data URI images already in the page consume the same downstream budget as the
// ones we generate, so reserve room for them before inlining any private image.
available := maxPageImages - countDataURIImages(content)
if available <= 0 {
return content
}
candidates := r.collect(content, available)
if len(candidates) == 0 {
return content
}
cache := r.downloadAll(ctx, candidates)
if len(cache) == 0 {
return content
}
inlined, inlinedBytes := 0, 0
budgetSpent := false
return imgTagRe.ReplaceAllStringFunc(content, func(tag string) string {
if inlined >= available || budgetSpent {
return tag
}
src, ok := extractImageSrc(tag)
if !ok {
return tag
}
action, abs := r.classify(src)
if action != imgDownload {
return tag
}
dataURI := cache[abs]
if dataURI == "" {
return tag
}
if inlinedBytes+len(dataURI) > maxPageInlineBytes {
budgetSpent = true
return tag
}
inlined++
inlinedBytes += len(dataURI)
return replaceImageSrc(tag, dataURI)
})
}
// collect returns the unique same-origin download candidates whose first
// occurrence falls within the remaining inline budget, in document order. Because
// Resolve writes at most budget data URIs, a URL first seen after that many
// occurrences can never be inlined, so it is not downloaded. Duplicates collapse to
// one entry, so a repeated image is fetched once yet may be inlined at every
// occurrence inside the budget.
func (r *assetResolver) collect(content string, budget int) []string {
tags := imgTagRe.FindAllString(content, -1)
order := make([]string, 0, len(tags))
seen := make(map[string]struct{}, len(tags))
occurrences := 0
for _, tag := range tags {
src, ok := extractImageSrc(tag)
if !ok {
continue
}
action, abs := r.classify(src)
if action != imgDownload {
continue
}
if occurrences >= budget {
break
}
occurrences++
if _, dup := seen[abs]; dup {
continue
}
seen[abs] = struct{}{}
order = append(order, abs)
}
return order
}
// countDataURIImages returns how many <img> tags in content already carry a
// data:image URI. The downstream docparser counts every data:image occurrence
// against the same per-page limit whether it came from the page or from us, so
// these must be reserved before the resolver inlines any private Confluence image.
func countDataURIImages(content string) int {
count := 0
for _, tag := range imgTagRe.FindAllString(content, -1) {
src, ok := extractImageSrc(tag)
if !ok {
continue
}
if strings.HasPrefix(strings.ToLower(src), "data:image/") {
count++
}
}
return count
}
// downloadAll fetches each candidate with bounded concurrency and returns a
// URL-keyed cache of the successful ones as data URIs, holding at most
// maxPageInlineBytes of them. Failures and images past the budget are logged
// (redacted) and omitted, so the rewrite pass leaves their original src in place.
func (r *assetResolver) downloadAll(ctx context.Context, urls []string) map[string]string {
cache := make(map[string]string, len(urls))
cachedBytes := 0
var mu sync.Mutex
var wg sync.WaitGroup
sem := make(chan struct{}, imageDownloadConcurrency)
for _, u := range urls {
if ctx.Err() != nil {
break
}
wg.Add(1)
sem <- struct{}{}
go func(abs string) {
defer wg.Done()
defer func() { <-sem }()
data, mime, err := r.client.downloadImage(ctx, abs)
if err != nil {
logImageFailure(ctx, abs, err)
return
}
uri := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
mu.Lock()
fits := cachedBytes+len(uri) <= maxPageInlineBytes
if fits {
cache[abs] = uri
cachedBytes += len(uri)
}
mu.Unlock()
if !fits {
logImageFailure(ctx, abs, errImagePageBudget)
}
}(u)
}
wg.Wait()
return cache
}
// classify decides whether src is a same-origin private candidate to download or
// should be left untouched, returning the normalized absolute URL for downloads.
func (r *assetResolver) classify(src string) (imgAction, string) {
s := strings.TrimSpace(src)
if s == "" || strings.HasPrefix(strings.ToLower(s), "data:") {
return imgKeep, ""
}
u, err := url.Parse(s)
if err != nil {
return imgKeep, ""
}
if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" {
// file:, javascript:, and any other scheme are never fetched.
return imgKeep, ""
}
if u.Host != "" {
if u.Scheme == "" {
// Protocol-relative //host/path: only same-origin is eligible.
base, berr := url.Parse(r.client.cfg.baseURL)
if berr != nil || !strings.EqualFold(base.Host, u.Host) {
return imgKeep, ""
}
} else if !r.client.sameOrigin(u) {
// Absolute external http(s): keep, never send Confluence credentials.
return imgKeep, ""
}
}
abs, err := r.client.resolveEndpoint(s)
if err != nil {
return imgKeep, ""
}
return imgDownload, abs
}
// logImageFailure records a redacted warning: host and path only, never the
// query (which may carry signed tokens), the Authorization header, or the body.
func logImageFailure(ctx context.Context, abs string, err error) {
host, path := abs, ""
if u, perr := url.Parse(abs); perr == nil {
host, path = u.Host, u.Path
}
reason := "download_failed"
switch {
case errors.Is(err, errImageTooLarge):
reason = "oversize"
case errors.Is(err, errImageNonImageContent):
reason = "non_image_content_type"
case errors.Is(err, errImageRedirectOffOrigin):
reason = "cross_origin_redirect"
case errors.Is(err, errImagePageBudget):
reason = "page_budget"
}
var api *apiError
if errors.As(err, &api) {
reason = "status_" + strconv.Itoa(api.status)
}
logger.Warnf(ctx, "[Confluence] failed to resolve image host=%s path=%s reason=%s", host, path, reason)
}
// extractImageSrc returns the HTML-unescaped src value of an <img> tag.
func extractImageSrc(tag string) (string, bool) {
m := srcAttrRe.FindStringSubmatch(tag)
if m == nil {
return "", false
}
value := strings.TrimSpace(m[2])
if len(value) >= 2 {
if q := value[0]; (q == '"' || q == '\'') && value[len(value)-1] == q {
value = value[1 : len(value)-1]
}
}
value = strings.TrimSpace(html.UnescapeString(value))
if value == "" {
return "", false
}
return value, true
}
// replaceImageSrc swaps only the src attribute value for a double-quoted data
// URI, preserving the rest of the tag byte-for-byte.
func replaceImageSrc(tag, dataURI string) string {
loc := srcAttrRe.FindStringSubmatchIndex(tag)
if loc == nil {
return tag
}
start, end := loc[4], loc[5]
return tag[:start] + `"` + dataURI + `"` + tag[end:]
}

View File

@ -0,0 +1,612 @@
package confluence
import (
"bytes"
"context"
"encoding/base64"
"errors"
"io"
"net/http"
"strings"
"sync"
"testing"
)
const testBaseURL = "https://confluence.test/wiki"
func newImageTestClient(rt roundTripper) *client {
return &client{
cfg: config{baseURL: testBaseURL, username: "reader", secret: "secret"},
http: &http.Client{Transport: rt},
}
}
func imageResp(status int, contentType string, body []byte) *http.Response {
header := make(http.Header)
if contentType != "" {
header.Set("Content-Type", contentType)
}
return &http.Response{
StatusCode: status,
Header: header,
Body: io.NopCloser(bytes.NewReader(body)),
}
}
func redirectResp(location string) *http.Response {
header := make(http.Header)
header.Set("Location", location)
return &http.Response{
StatusCode: http.StatusFound,
Header: header,
Body: io.NopCloser(bytes.NewReader(nil)),
}
}
// reqLog records outbound requests. Image downloads run concurrently, so every
// access is mutex-guarded.
type reqLog struct {
mu sync.Mutex
reqs []reqRecord
}
type reqRecord struct {
host string
path string
hasAuth bool
}
func newReqLog() *reqLog { return &reqLog{} }
func (l *reqLog) add(req *http.Request) {
l.mu.Lock()
defer l.mu.Unlock()
l.reqs = append(l.reqs, reqRecord{
host: req.URL.Host,
path: req.URL.Path,
hasAuth: req.Header.Get("Authorization") != "",
})
}
func (l *reqLog) count() int {
l.mu.Lock()
defer l.mu.Unlock()
return len(l.reqs)
}
func (l *reqLog) all() []reqRecord {
l.mu.Lock()
defer l.mu.Unlock()
return append([]reqRecord(nil), l.reqs...)
}
func (l *reqLog) hasHost(host string) bool {
for _, r := range l.all() {
if r.host == host {
return true
}
}
return false
}
func (l *reqLog) hasPath(path string) bool {
for _, r := range l.all() {
if r.path == path {
return true
}
}
return false
}
func (l *reqLog) anyWithoutAuth() bool {
for _, r := range l.all() {
if !r.hasAuth {
return true
}
}
return false
}
var testPNG = []byte("private-png-bytes")
func testDataURI(body []byte) string {
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(body)
}
func TestAssetResolverKeepsPublicURL(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<p>hi</p><img src="https://cdn.example.com/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("public URL was modified:\n%s", out)
}
if log.count() != 0 {
t.Fatalf("public image was downloaded: %v", log.all())
}
}
func TestAssetResolverKeepsBase64(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="data:image/png;base64,QUJDRA==">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("base64 image was modified:\n%s", out)
}
if log.count() != 0 {
t.Fatalf("base64 image triggered a download: %v", log.all())
}
}
func TestAssetResolverKeepsNonHTTPScheme(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="file:///etc/passwd"><img src="javascript:void(0)">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("non-http scheme was modified:\n%s", out)
}
if log.count() != 0 {
t.Fatalf("non-http scheme triggered a download: %v", log.all())
}
}
func TestAssetResolverInlinesRelativeConfluenceImage(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="/wiki/download/attachments/123/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
want := testDataURI(testPNG)
if !strings.Contains(out, `src="`+want+`"`) {
t.Fatalf("relative image not inlined as data URI:\n%s", out)
}
if strings.Contains(out, "/wiki/download/attachments") {
t.Fatalf("original private URL survived:\n%s", out)
}
reqs := log.all()
if len(reqs) != 1 {
t.Fatalf("expected 1 download, got %d: %v", len(reqs), reqs)
}
if reqs[0].host != "confluence.test" || !reqs[0].hasAuth {
t.Fatalf("download missing credentials or wrong host: %+v", reqs[0])
}
}
func TestAssetResolverInlinesSameOriginPrivateImage(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="https://confluence.test/wiki/download/attachments/123/a.png" alt="x"/>`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if !strings.Contains(out, `src="`+testDataURI(testPNG)+`"`) {
t.Fatalf("same-origin image not inlined:\n%s", out)
}
// The rest of the tag must be preserved.
if !strings.Contains(out, `alt="x"`) {
t.Fatalf("sibling attribute lost:\n%s", out)
}
if log.count() != 1 {
t.Fatalf("expected 1 download, got %d", log.count())
}
}
func TestAssetResolverDoesNotSendCredentialsCrossOrigin(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="https://cdn.example.com/a.png">` +
`<img src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if log.hasHost("cdn.example.com") {
t.Fatalf("external host received a request: %v", log.all())
}
if !strings.Contains(out, "https://cdn.example.com/a.png") {
t.Fatalf("external URL was modified:\n%s", out)
}
if !strings.Contains(out, testDataURI(testPNG)) {
t.Fatalf("same-origin image was not inlined:\n%s", out)
}
if log.count() != 1 || log.anyWithoutAuth() {
t.Fatalf("expected exactly one authenticated same-origin download: %v", log.all())
}
}
func TestAssetResolverRejectsCrossOriginRedirect(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
if req.URL.Host == "confluence.test" {
return redirectResp("https://evil.example.com/a.png"), nil
}
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("cross-origin redirect was followed and inlined:\n%s", out)
}
if log.hasHost("evil.example.com") {
t.Fatalf("credentials could have leaked to evil host: %v", log.all())
}
if log.count() != 1 {
t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all())
}
}
func TestAssetResolverRejectsSameOriginRedirectOutsideContextPath(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
// The base URL is https://confluence.test/wiki. A redirect to another app on
// the same host stays same-origin but escapes the /wiki context path, and Go
// keeps sending Basic Auth on a same-host hop. It must be refused before the
// credentialed request is ever sent, not merely discarded afterward.
if strings.HasPrefix(req.URL.Path, "/wiki/") {
return redirectResp("https://confluence.test/other-app/leak.png"), nil
}
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("same-host redirect outside the context path was followed and inlined:\n%s", out)
}
if log.hasPath("/other-app/leak.png") {
t.Fatalf("credentials followed a redirect outside the context path: %v", log.all())
}
if log.count() != 1 {
t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all())
}
}
func TestAssetResolverRejectsOversizedImage(t *testing.T) {
log := newReqLog()
big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1)
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", big), nil
})
in := `<img src="/wiki/download/attachments/1/big.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("oversized image was inlined:\n%s", out)
}
if strings.Contains(out, "data:image") {
t.Fatalf("oversized image produced a data URI")
}
}
func TestAssetResolverRejectsNonImageResponse(t *testing.T) {
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(http.StatusOK, "text/html; charset=utf-8", []byte("<html>login required</html>")), nil
})
in := `<img src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("login HTML was inlined as an image:\n%s", out)
}
}
func TestAssetResolverKeepsOriginalURLOnDownloadFailure(t *testing.T) {
for _, status := range []int{http.StatusForbidden, http.StatusNotFound, http.StatusInternalServerError} {
t.Run(http.StatusText(status), func(t *testing.T) {
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(status, "image/png", testPNG), nil
})
in := `<img src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("status %d modified the document:\n%s", status, out)
}
})
}
}
func TestAssetResolverDeduplicatesSamePageImages(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
src := `<img src="/wiki/download/attachments/1/a.png">`
in := strings.Repeat(src, 5)
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if got := strings.Count(out, "data:image/png;base64,"); got != 5 {
t.Fatalf("expected 5 inlined copies, got %d:\n%s", got, out)
}
if log.count() != 1 {
t.Fatalf("duplicate src downloaded %d times, want 1", log.count())
}
}
func TestAssetResolverRespectsPerPageImageCap(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
var sb strings.Builder
total := maxPageImages + 5
for i := 0; i < total; i++ {
sb.WriteString(`<img src="/wiki/download/attachments/1/img`)
sb.WriteString(strings.Repeat("x", i+1)) // unique path per image
sb.WriteString(`.png">`)
}
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String())
if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages {
t.Fatalf("inlined %d images, want cap %d", got, maxPageImages)
}
if log.count() != maxPageImages {
t.Fatalf("downloaded %d images, want cap %d", log.count(), maxPageImages)
}
if got := strings.Count(out, "/wiki/download/attachments"); got != 5 {
t.Fatalf("expected 5 images past the cap to keep original src, got %d", got)
}
}
func TestAssetResolverCapsOccurrencesNotUniqueDownloads(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
// Two unique URLs but 35 occurrences. Counting the cap by unique URL would
// download cold.png too and inline 30 + 5 data URIs, overflowing the downstream
// per-page limit that counts every occurrence. The budget must be occurrences.
hot := `<img src="/wiki/download/attachments/1/hot.png">`
cold := `<img src="/wiki/download/attachments/1/cold.png">`
in := strings.Repeat(hot, maxPageImages) + strings.Repeat(cold, 5)
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages {
t.Fatalf("inlined %d data URIs, want cap %d", got, maxPageImages)
}
if got := strings.Count(out, "/wiki/download/attachments/1/cold.png"); got != 5 {
t.Fatalf("expected the 5 images past the cap to keep original src, got %d", got)
}
// The repeated image downloads once; the image past the budget never downloads.
if log.count() != 1 {
t.Fatalf("downloaded %d images, want 1 (dedup + no over-budget fetch): %v", log.count(), log.all())
}
if log.hasPath("/wiki/download/attachments/1/cold.png") {
t.Fatalf("image past the occurrence cap was downloaded: %v", log.all())
}
}
// largeTestImage is just under maxImageBytes, so a handful of them overflow
// maxPageInlineBytes long before the per-page occurrence cap.
var largeTestImage = bytes.Repeat([]byte("a"), int(maxImageBytes)-1024)
func TestAssetResolverStopsInliningPastPageByteBudget(t *testing.T) {
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(http.StatusOK, "image/png", largeTestImage), nil
})
want := maxPageInlineBytes / len(testDataURI(largeTestImage))
total := want + 3
src := `<img src="/wiki/download/attachments/1/big.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), strings.Repeat(src, total))
if got := strings.Count(out, "data:image/png;base64,"); got != want {
t.Fatalf("inlined %d copies, want %d within the %d-byte page budget", got, want, maxPageInlineBytes)
}
if got := strings.Count(out, "/wiki/download/attachments/1/big.png"); got != total-want {
t.Fatalf("expected %d copies past the budget to keep original src, got %d", total-want, got)
}
}
func TestAssetResolverDownloadCacheRespectsPageByteBudget(t *testing.T) {
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(http.StatusOK, "image/png", largeTestImage), nil
})
want := maxPageInlineBytes / len(testDataURI(largeTestImage))
urls := make([]string, 0, want+3)
for i := 0; i < want+3; i++ {
urls = append(urls, testBaseURL+"/download/attachments/1/img"+strings.Repeat("x", i+1)+".png")
}
cache := newAssetResolver(newImageTestClient(rt)).downloadAll(context.Background(), urls)
if len(cache) != want {
t.Fatalf("cached %d images, want %d within the %d-byte page budget", len(cache), want, maxPageInlineBytes)
}
}
func TestAssetResolverSharesCapWithExistingDataURIImages(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
// Data URI images already in the page share the downstream 30-image budget with
// the ones we generate. With 10 existing, only 20 of the 25 private images may
// be inlined; the rest keep their original URL instead of overflowing the cap.
var sb strings.Builder
for i := 0; i < 10; i++ {
sb.WriteString(`<img src="data:image/gif;base64,EXISTING">`)
}
for i := 0; i < 25; i++ {
sb.WriteString(`<img src="/wiki/download/attachments/1/img`)
sb.WriteString(strings.Repeat("x", i+1)) // unique path per image
sb.WriteString(`.png">`)
}
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String())
// The 10 pre-existing data URIs are preserved untouched.
if got := strings.Count(out, "data:image/gif;base64,EXISTING"); got != 10 {
t.Fatalf("preserved %d existing data URIs, want 10", got)
}
// Exactly 20 private images inlined: 30 shared budget - 10 already present.
if got := strings.Count(out, testDataURI(testPNG)); got != 20 {
t.Fatalf("inlined %d private images, want 20", got)
}
// The 5 private images past the shared budget keep their original URL.
if got := strings.Count(out, "/wiki/download/attachments"); got != 5 {
t.Fatalf("expected 5 private images past the cap to keep original src, got %d", got)
}
// Only the 20 inlinable private images are downloaded.
if log.count() != 20 {
t.Fatalf("downloaded %d images, want 20: %v", log.count(), log.all())
}
}
func TestAssetResolverIgnoresDataSrcAttribute(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
// data-src must not be mistaken for src; there is no real src here.
in := `<img data-src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if out != in {
t.Fatalf("data-src was treated as src:\n%s", out)
}
if log.count() != 0 {
t.Fatalf("data-src triggered a download: %v", log.all())
}
}
func TestAssetResolverToleratesGreaterThanInAttribute(t *testing.T) {
log := newReqLog()
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
log.add(req)
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img alt="a > b" src="/wiki/download/attachments/1/a.png">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if !strings.Contains(out, testDataURI(testPNG)) {
t.Fatalf("image with '>' in alt was not inlined:\n%s", out)
}
if !strings.Contains(out, `alt="a > b"`) {
t.Fatalf("alt attribute was corrupted:\n%s", out)
}
}
func TestAssetResolverUnescapesAmpersandInQuery(t *testing.T) {
var gotPath string
var gotQuery string
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
gotPath = req.URL.Path
gotQuery = req.URL.RawQuery
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
in := `<img src="/wiki/download/attachments/1/a.png?api=v2&amp;v=3">`
out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
if !strings.Contains(out, "data:image/png;base64,") {
t.Fatalf("image with &amp; query was not inlined:\n%s", out)
}
if gotPath != "/wiki/download/attachments/1/a.png" || gotQuery != "api=v2&v=3" {
t.Fatalf("query not unescaped: path=%q query=%q", gotPath, gotQuery)
}
}
func TestDownloadPrivateImageLimitsBodySize(t *testing.T) {
big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1)
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(http.StatusOK, "image/png", big), nil
})
c := newImageTestClient(rt)
_, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/big.png")
if !errors.Is(err, errImageTooLarge) {
t.Fatalf("downloadImage() error = %v, want errImageTooLarge", err)
}
}
func TestDownloadPrivateImageValidatesContentType(t *testing.T) {
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(http.StatusOK, "application/json", []byte(`{"error":"unauthorized"}`)), nil
})
c := newImageTestClient(rt)
_, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png")
if !errors.Is(err, errImageNonImageContent) {
t.Fatalf("downloadImage() error = %v, want errImageNonImageContent", err)
}
}
func TestDownloadPrivateImageSniffsWhenContentTypeMissing(t *testing.T) {
png := append([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, bytes.Repeat([]byte{0}, 64)...)
rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
return imageResp(http.StatusOK, "", png), nil
})
c := newImageTestClient(rt)
body, mime, err := c.downloadImage(
context.Background(),
"https://confluence.test/wiki/download/attachments/1/a.png",
)
if err != nil {
t.Fatalf("downloadImage() error = %v", err)
}
if mime != "image/png" {
t.Fatalf("sniffed mime = %q, want image/png", mime)
}
if !bytes.Equal(body, png) {
t.Fatalf("body mismatch: %d bytes", len(body))
}
}
func TestDownloadPrivateImageRejectsCrossOriginRedirect(t *testing.T) {
rt := roundTripper(func(req *http.Request) (*http.Response, error) {
if req.URL.Host == "confluence.test" {
return redirectResp("https://evil.example.com/a.png"), nil
}
return imageResp(http.StatusOK, "image/png", testPNG), nil
})
c := newImageTestClient(rt)
_, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png")
if !errors.Is(err, errImageRedirectOffOrigin) {
t.Fatalf("downloadImage() error = %v, want errImageRedirectOffOrigin", err)
}
}
// TestFetchStreamInlinesPrivateImageAsDataURI is the connector integration test:
// a body.view containing a same-origin private image flows through FetchStream
// and the emitted FetchedItem.Content carries an inline data URI (which the
// downstream docparser pipeline later persists and rewrites to resource://).
func TestFetchStreamInlinesPrivateImageAsDataURI(t *testing.T) {
api := &streamAPI{
pages: []streamPage{{id: "p1", title: "Page", version: 1}},
bodyHTML: map[string]string{"p1": `<p>hi</p><img src="/wiki/download/attachments/p1/a.png">`},
}
h := &captureHandler{}
if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), nil, h); err != nil {
t.Fatal(err)
}
if len(h.items) != 1 {
t.Fatalf("emitted %d items, want 1", len(h.items))
}
content := string(h.items[0].Content)
if !strings.Contains(content, "data:image/png;base64,") {
t.Fatalf("private image was not inlined into Content:\n%s", content)
}
if strings.Contains(content, "/wiki/download/attachments") {
t.Fatalf("original private Confluence URL leaked into Content:\n%s", content)
}
if api.imageCalls != 1 {
t.Fatalf("image download calls = %d, want 1", api.imageCalls)
}
}

View File

@ -17,19 +17,29 @@ import (
)
const (
maxJSONResponseBytes int64 = 20 << 20
requestAttempts = 4
maxErrorResponseRunes = 1000
maxRetryDelay = 60 * time.Second
maxPaginationHops = 10000
maxJSONResponseBytes int64 = 20 << 20
maxImageBytes int64 = 10 << 20
maxImageRedirects = 10
requestAttempts = 4
maxErrorResponseRunes = 1000
maxRetryDelay = 60 * time.Second
maxPaginationHops = 10000
maxTraversalNodes = 100000
maxTransparentTraversalNodes = 10000
maxTransparentTraversalDepth = 32
// maxEmptyServerPages bounds how far a Server/Data Center page listing
// follows _links.next through empty pages. Some builds keep returning
// next past the end of a space (#3596); permission filtering can also
// empty one page mid-listing, so a single empty page is not the end.
// follows _links.next through empty pages.
maxEmptyServerPages = 3
// maxDeletionProbes caps how many unlisted pages one sync looks up
// individually after a cut-short listing; the rest wait for the next run.
maxDeletionProbes = 200
maxDeletionProbes = 200
)
var (
errImageTooLarge = errors.New("confluence image exceeds size limit")
errImageNonImageContent = errors.New("confluence image response is not an image")
// errImageRedirectOffOrigin fires whenever a redirect target leaves the base
// URL boundary resolveEndpoint enforces — a different origin, or the same host
// but a path outside the configured context path (e.g. /wiki → /other-app).
errImageRedirectOffOrigin = errors.New("confluence image redirect left the configured base URL")
)
type apiError struct {
@ -232,6 +242,121 @@ func (c *client) resourceURL(link string) string {
return resolved
}
// sameOrigin reports whether u shares scheme and host with the configured base
// URL. Only same-origin URLs may carry Confluence credentials.
func (c *client) sameOrigin(u *url.URL) bool {
if u == nil {
return false
}
base, err := url.Parse(c.cfg.baseURL)
if err != nil {
return false
}
return strings.EqualFold(base.Scheme, u.Scheme) && strings.EqualFold(base.Host, u.Host)
}
// withinImageScope reports whether u stays inside the boundary resolveEndpoint
// enforces for the initial image URL: the same origin AND a path inside the
// configured context path. Redirect targets are validated with this same rule,
// not merely sameOrigin, because Go forwards Basic Auth on a same-host redirect —
// so a hop like /wiki/download → /other-app/image must fail closed too.
func (c *client) withinImageScope(u *url.URL) bool {
if u == nil {
return false
}
_, err := c.resolveEndpoint(u.String())
return err == nil
}
// downloadImage fetches a same-origin private image using the connector's
// credentials. It streams with a hard size cap, fails closed if a redirect left
// the Confluence origin, and validates the response really is an image. It is a
// best-effort single attempt: a failed image degrades to keeping its original
// URL rather than retrying or failing the page.
func (c *client) downloadImage(ctx context.Context, absURL string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, absURL, nil)
if err != nil {
return nil, "", err
}
req.SetBasicAuth(c.cfg.username, c.cfg.secret)
req.Header.Set("Accept", "image/*,*/*;q=0.8")
resp, err := c.imageHTTPClient().Do(req)
if err != nil {
return nil, "", err
}
defer func() { _ = resp.Body.Close() }()
// Backstop for the redirect policy: imageHTTPClient's CheckRedirect already
// refuses any hop that leaves the base before it is followed, so this only
// re-verifies the final URL when the transport populated resp.Request. A custom
// RoundTripper may leave it nil, in which case there is no followed redirect to
// re-check and the download is allowed to proceed.
if resp.Request != nil && !c.withinImageScope(resp.Request.URL) {
return nil, "", errImageRedirectOffOrigin
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, "", &apiError{endpoint: "image", status: resp.StatusCode}
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxImageBytes+1))
if err != nil {
return nil, "", err
}
if int64(len(body)) > maxImageBytes {
return nil, "", errImageTooLarge
}
mime, ok := imageContentType(resp.Header.Get("Content-Type"), body)
if !ok {
return nil, "", errImageNonImageContent
}
return body, mime, nil
}
// imageHTTPClient reuses the connector's SSRF-safe transport and timeout but
// pins redirects to the configured Confluence base, so a private-image fetch can
// never be bounced to a third party or to another app on the same host. The
// redirect check fails closed: any hop that resolveEndpoint would reject — a
// foreign origin, or a same-host path outside the context path (which Go would
// still send Basic Auth to) — is refused outright rather than merely stripped of
// credentials.
func (c *client) imageHTTPClient() *http.Client {
return &http.Client{
Transport: c.http.Transport,
Timeout: c.http.Timeout,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= maxImageRedirects {
return errors.New("confluence image stopped after too many redirects")
}
if !c.withinImageScope(req.URL) {
return errImageRedirectOffOrigin
}
return nil
},
}
}
// imageContentType normalizes a response Content-Type to an image MIME type. An
// empty or generic binary type falls back to content sniffing; any explicit
// non-image type (for example the text/html login page Confluence returns when a
// token expired) is rejected so it is never stored as an image.
func imageContentType(header string, body []byte) (string, bool) {
ct := strings.ToLower(strings.TrimSpace(header))
if i := strings.Index(ct, ";"); i >= 0 {
ct = strings.TrimSpace(ct[:i])
}
switch {
case strings.HasPrefix(ct, "image/"):
return ct, true
case ct != "" && ct != "application/octet-stream":
return "", false
}
if sniff := strings.ToLower(http.DetectContentType(body)); strings.HasPrefix(sniff, "image/") {
if i := strings.Index(sniff, ";"); i >= 0 {
sniff = strings.TrimSpace(sniff[:i])
}
return sniff, true
}
return "", false
}
func (c *client) ping(ctx context.Context) error {
if c.cfg.cloud() {
return c.get(ctx, "/api/v2/spaces?limit=1", nil)
@ -308,10 +433,6 @@ func withCloudPageQuery(next string) string {
return parsed.String()
}
// pages lists every current page in a space. complete is false when a Server
// listing was cut short after maxEmptyServerPages empty pages that still
// linked onward; the caller must not treat pages missing from such a listing
// as deleted.
func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {
if c.cfg.cloud() {
var all []page
@ -371,10 +492,6 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo
return all, complete, err
}
// pageInSpace reports whether a Server page is still a current page of the
// space. Confluence answers 404 for a page that was deleted or moved to the
// trash; a page that is no longer current or now belongs to another space is
// gone from this one too, matching what a complete listing would omit.
func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) {
var result struct {
Status string `json:"status"`
@ -406,3 +523,349 @@ func (c *client) body(ctx context.Context, id string) (pageBody, error) {
err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=body.view,version,space", &result)
return result, err
}
// pageDetail obtains identity, ownership, and version without requiring a
// rendered body. Scope enumeration must remain complete when a body is
// temporarily unavailable, otherwise a transient content error would be
// indistinguishable from a missing page during deletion reconciliation.
func (c *client) pageDetail(ctx context.Context, id string) (page, error) {
var result page
if c.cfg.cloud() {
err := c.get(ctx, "/api/v2/pages/"+url.PathEscape(id), &result)
return result, err
}
err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=version,space", &result)
return result, err
}
// validatePageOwnership performs the constant-cost check required before a
// picker expansion. Ancestors are deliberately not fetched here: they are only
// needed by ResolveResourceAncestors and buildSyncPlan.
func (c *client) validatePageOwnership(ctx context.Context, s space, pageID string) error {
full, err := c.pageDetail(ctx, pageID)
if err != nil {
return err
}
if c.cfg.cloud() {
if full.SpaceID != s.ID {
return fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID)
}
return nil
}
if full.Space.Key == "" || full.Space.Key != s.Key {
return fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
}
return nil
}
func serverChildPagesEndpoint(pageID string) string {
query := url.Values{"expand": []string{serverPageExpand}, "limit": []string{"100"}}
return "/rest/api/content/" + url.PathEscape(pageID) + "/child/page?" + query.Encode()
}
func serverTopLevelPagesEndpoint(spaceKey string) string {
query := url.Values{"depth": []string{"root"}, "expand": []string{serverPageExpand}, "limit": []string{"100"}}
return "/rest/api/space/" + url.PathEscape(spaceKey) + "/content/page?" + query.Encode()
}
func (c *client) listServerPages(ctx context.Context, endpoint string) ([]page, error) {
start, _ := url.Parse(endpoint)
startQuery := start.Query()
var all []page
err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) {
var result pageList
if err := c.get(ctx, pageURL, &result); err != nil {
return "", err
}
all = append(all, result.Results...)
next := result.Links.Next
if next != "" {
next = withServerListQuery(next, startQuery)
}
return next, nil
})
return all, err
}
// Server pagination links commonly retain only start/limit. Restore the query
// that defines the listing scope (notably depth=root) as well as expand.
func withServerListQuery(next string, required url.Values) string {
parsed, err := url.Parse(next)
if err != nil {
return next
}
query := parsed.Query()
for key, values := range required {
if query.Get(key) == "" && len(values) > 0 {
query[key] = values
}
}
parsed.RawQuery = query.Encode()
return parsed.String()
}
func (c *client) listCloudHierarchy(ctx context.Context, endpoint string) ([]page, error) {
var all []page
err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) {
var result cloudHierarchyList
if err := c.get(ctx, pageURL, &result); err != nil {
return "", err
}
for _, item := range result.Results {
all = append(
all,
page{
ID: item.ID,
Title: item.Title,
Status: item.Status,
Kind: item.Type,
SpaceID: item.SpaceID,
Links: item.Links,
},
)
}
return result.Links.Next, nil
})
return all, err
}
// topLevelPages retrieves one level only. The Server/DC depth=root endpoint
// is paginated by the server; it deliberately never falls back to scanning all
// pages because that would defeat lazy selection for large spaces.
//
// Cloud applies the same lazy contract: depth=0 returns only root-level pages,
// and Cloud exposes no endpoint to enumerate a space's top-level folders
// (CONFCLOUD-84275), so pages nested directly under those containers stay
// unselectable in the picker for now. Recovering them would require a
// whole-space scan per expand, which is exactly what lazy loading forbids;
// selecting the whole space still syncs them. The connector flags this
// limitation through resource metadata so the picker can surface it.
func (c *client) topLevelPages(ctx context.Context, s space) ([]page, error) {
var pages []page
var err error
if c.cfg.cloud() {
pages, err = c.listCloudHierarchy(
ctx,
"/api/v2/spaces/"+url.PathEscape(s.ID)+"/pages?status=current&depth=0&limit=250",
)
if err == nil {
visible := pages[:0]
for _, page := range pages {
if page.Kind == "" || page.Kind == "page" {
visible = append(visible, page)
}
}
pages = visible
}
} else {
pages, err = c.listServerPages(ctx, serverTopLevelPagesEndpoint(s.Key))
}
if err == nil {
return pages, nil
}
// The depth=root navigation hint is Server/DC-specific; a Cloud 400/404 has
// different causes and must surface the raw API error for diagnosis.
if !c.cfg.cloud() {
var api *apiError
if errors.As(err, &api) && (api.status == http.StatusBadRequest || api.status == http.StatusNotFound) {
return nil, fmt.Errorf(
"confluence server does not support complete top-level page navigation for this space; "+
"select the Space to sync all pages: %w",
err,
)
}
}
return nil, err
}
func (c *client) directChildPages(ctx context.Context, _ space, pageID string) ([]page, error) {
if c.cfg.cloud() {
return c.visibleCloudPageChildren(ctx, pageID)
}
return c.listServerPages(ctx, serverChildPagesEndpoint(pageID))
}
func cloudDirectChildrenEndpoint(kind, id string) (string, error) {
plural := map[string]string{
"page": "pages", "folder": "folders", "database": "databases", "whiteboard": "whiteboards", "embed": "embeds",
}[kind]
if plural == "" {
return "", fmt.Errorf("unsupported Confluence Cloud hierarchy type %q", kind)
}
return "/api/v2/" + plural + "/" + url.PathEscape(id) + "/direct-children?limit=250", nil
}
// visibleCloudPageChildren projects Cloud's mixed-content hierarchy onto a
// Page-only picker. Containers remain connector-internal and are traversed
// until the next visible Page is found.
func (c *client) visibleCloudPageChildren(ctx context.Context, pageID string) ([]page, error) {
return c.walkCloudPageHierarchy(ctx, pageID, false)
}
// walkCloudPageHierarchy traverses Cloud's mixed content tree through
// direct-children endpoints. Picker expansion stops at visible pages, while a
// sync walk continues through them to return the complete page subtree.
func (c *client) walkCloudPageHierarchy(ctx context.Context, pageID string, descendPages bool) ([]page, error) {
type queuedNode struct {
id, kind string
depth int
}
queue := []queuedNode{{id: pageID, kind: "page"}}
visited := map[string]struct{}{"page:" + pageID: {}}
pages := make([]page, 0)
nodeLimit := maxTransparentTraversalNodes
if descendPages {
nodeLimit = maxTraversalNodes
}
for len(queue) > 0 {
current := queue[0]
queue = queue[1:]
endpoint, err := cloudDirectChildrenEndpoint(current.kind, current.id)
if err != nil {
return nil, err
}
children, err := c.listCloudHierarchy(ctx, endpoint)
if err != nil {
return nil, err
}
for _, child := range children {
kind := child.Kind
if kind == "" {
kind = "page"
}
key := kind + ":" + child.ID
if child.ID == "" || (kind == "page" && child.ID == pageID) {
continue
}
if _, exists := visited[key]; exists {
continue
}
if len(visited) >= nodeLimit {
return nil, fmt.Errorf(
"confluence Cloud hierarchy traversal exceeds %d nodes",
nodeLimit,
)
}
visited[key] = struct{}{}
if kind == "page" {
pages = append(pages, child)
if !descendPages {
continue
}
}
depth := 0
if kind != "page" {
depth = current.depth + 1
if current.kind == "page" {
depth = 1
}
}
if depth > maxTransparentTraversalDepth {
return nil, fmt.Errorf(
"confluence Cloud transparent traversal exceeds depth %d",
maxTransparentTraversalDepth,
)
}
if _, err := cloudDirectChildrenEndpoint(kind, child.ID); err != nil {
return nil, err
}
queue = append(queue, queuedNode{id: child.ID, kind: kind, depth: depth})
}
}
return pages, nil
}
// pageAncestors validates that pageID belongs to s and returns its path from
// the space root. A selected page is fetched even if it is top-level, so a
// forged page:{space}:{page} ID cannot cross a space boundary.
func (c *client) pageAncestors(ctx context.Context, s space, pageID string) ([]page, error) {
if c.cfg.cloud() {
full, err := c.pageDetail(ctx, pageID)
if err != nil {
return nil, err
}
if full.SpaceID != s.ID {
return nil, fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID)
}
ancestors, err := c.listCloudHierarchy(ctx, "/api/v2/pages/"+url.PathEscape(pageID)+"/ancestors?limit=250")
if err != nil {
return nil, err
}
return ancestors, nil
}
var full page
if err := c.get(ctx, "/rest/api/content/"+url.PathEscape(pageID)+"?expand=ancestors,space", &full); err != nil {
return nil, err
}
if full.Space.Key == "" || full.Space.Key != s.Key {
return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
}
return full.Ancestors, nil
}
// pageSubtree uses direct-child traversal for Server/DC because that endpoint
// is stable across supported versions. Cloud uses the same complete
// mixed-content hierarchy walk as the picker, but continues through visible
// pages to include every page in the selected subtree.
func (c *client) pageSubtree(ctx context.Context, s space, pageID string) ([]page, error) {
full, err := c.pageDetail(ctx, pageID)
if err != nil {
return nil, err
}
if c.cfg.cloud() {
if full.SpaceID != s.ID {
return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
}
} else if full.Space.Key == "" || full.Space.Key != s.Key {
return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
}
root := full
if c.cfg.cloud() {
descendants, err := c.walkCloudPageHierarchy(ctx, pageID, true)
if err != nil {
return nil, err
}
out := []page{root}
for _, descendant := range descendants {
// Direct-child responses are intentionally compact. Hydrate each page
// before version comparison; an unknown version must never be treated
// as an unchanged stable token.
detail, err := c.pageDetail(ctx, descendant.ID)
if err != nil {
return nil, err
}
if detail.SpaceID != s.ID {
return nil, fmt.Errorf("page %s does not belong to space %s", descendant.ID, s.ID)
}
out = append(out, detail)
}
return out, nil
}
queue, out := []page{root}, []page{root}
seen := map[string]bool{root.ID: true}
for len(queue) > 0 {
current := queue[0]
queue = queue[1:]
children, err := c.directChildPages(ctx, s, current.ID)
if err != nil {
return nil, err
}
for _, child := range children {
if child.ID == "" || seen[child.ID] {
continue
}
if len(seen) >= maxTraversalNodes {
return nil, fmt.Errorf("confluence page subtree exceeds %d nodes", maxTraversalNodes)
}
seen[child.ID] = true
if child.SpaceID != "" && child.SpaceID != s.ID {
return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID)
}
if !c.cfg.cloud() && child.Space.Key != "" && child.Space.Key != s.Key {
return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID)
}
queue, out = append(queue, child), append(out, child)
}
}
return out, nil
}

View File

@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
@ -145,6 +146,70 @@ func TestServerSpacePageListPrefersFlatResults(t *testing.T) {
}
}
func TestTopLevelPagesServerRestoresScopeOnNext(t *testing.T) {
var depths, expands []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/rest/api/space/ENG/content/page" {
http.NotFound(w, r)
return
}
depths = append(depths, r.URL.Query().Get("depth"))
expands = append(expands, r.URL.Query().Get("expand"))
w.Header().Set("Content-Type", "application/json")
if r.URL.Query().Get("start") == "100" {
_, _ = w.Write([]byte(`{
"results": [{"id": "2", "title": "Second", "version": {"number": 2}}]
}`))
return
}
_, _ = w.Write([]byte(`{
"_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"},
"results": [{"id": "1", "title": "Root", "version": {"number": 1}}]
}`))
}))
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
pages, err := c.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
if len(pages) != 2 || pages[0].ID != "1" || pages[1].ID != "2" {
t.Fatalf("pages = %#v", pages)
}
// The server's _links.next drops depth and expand; the connector must
// rebuild the listing scope or later pages would look versionless.
if len(depths) != 2 || depths[0] != "root" || depths[1] != "root" {
t.Fatalf("depth query = %#v", depths)
}
if len(expands) != 2 || expands[0] != serverPageExpand || expands[1] != serverPageExpand {
t.Fatalf("expand query = %#v", expands)
}
}
func TestTopLevelPagesCloudSurfacesRawAPIError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"message":"space not found"}`))
}))
t.Cleanup(server.Close)
cloud := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}
_, err := cloud.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err == nil {
t.Fatal("cloud topLevelPages unexpectedly succeeded")
}
if strings.Contains(err.Error(), "does not support") {
t.Fatalf("cloud 404 reused the Server/DC navigation hint: %v", err)
}
srv := &client{cfg: config{baseURL: server.URL + "/wiki"}, http: server.Client()}
_, err = srv.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err == nil || !strings.Contains(err.Error(), "confluence server does not support") {
t.Fatalf("server error = %v; want the Server/DC navigation hint", err)
}
}
func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
var depths []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@ -3,10 +3,14 @@ package confluence
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"net/http"
"sort"
"strings"
"sync"
"time"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/datasource"
@ -21,12 +25,64 @@ var (
// Connector implements datasource.StreamingConnector for Confluence.
type Connector struct {
newClient func(config) (*client, error)
newClient func(config) (*client, error)
spaceCache spaceCache
}
const pickerSpaceCacheTTL = time.Minute
type spaceCache struct {
mu sync.Mutex
entries map[string]cachedSpaces
}
type cachedSpaces struct {
values []space
expires time.Time
}
// NewConnector creates a Confluence connector.
func NewConnector() *Connector { return &Connector{newClient: newClient} }
func (c *Connector) pickerSpaces(ctx context.Context, client *client, cfg config) ([]space, error) {
key := pickerSpaceCacheKey(cfg)
now := time.Now()
c.spaceCache.mu.Lock()
if cached, ok := c.spaceCache.entries[key]; ok && now.Before(cached.expires) {
values := append([]space(nil), cached.values...)
c.spaceCache.mu.Unlock()
return values, nil
}
c.spaceCache.mu.Unlock()
spaces, err := client.spaces(ctx)
if err != nil {
return nil, err
}
c.spaceCache.mu.Lock()
if c.spaceCache.entries == nil {
c.spaceCache.entries = make(map[string]cachedSpaces)
}
for key, cached := range c.spaceCache.entries {
if !now.Before(cached.expires) {
delete(c.spaceCache.entries, key)
}
}
c.spaceCache.entries[key] = cachedSpaces{
values: append([]space(nil), spaces...),
expires: now.Add(pickerSpaceCacheTTL),
}
c.spaceCache.mu.Unlock()
return spaces, nil
}
// pickerSpaceCacheKey partitions cached space visibility by every credential
// component that can affect permissions. The secret is fingerprinted so it
// never becomes a map key retained in process memory or appears in diagnostics.
func pickerSpaceCacheKey(cfg config) string {
fingerprint := sha256.Sum256([]byte(cfg.secret))
return cfg.edition + "\x00" + cfg.baseURL + "\x00" + cfg.username + "\x00" + fmt.Sprintf("%x", fingerprint)
}
// Type returns the connector type identifier.
func (*Connector) Type() string { return types.ConnectorTypeConfluence }
@ -52,13 +108,95 @@ func (c *Connector) Validate(ctx context.Context, ds *types.DataSourceConfig) er
return client.ping(ctx)
}
// ListResources returns the Confluence spaces the credentials can see.
// ListResources exposes a lazy Space → Page tree. It never scans a whole
// space merely to render the picker.
func (c *Connector) ListResources(
ctx context.Context, ds *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
if parentID != "" {
return nil, nil
client, cfg, err := c.configured(ds)
if err != nil {
return nil, err
}
if parentID == "" {
spaces, err := c.pickerSpaces(ctx, client, cfg)
if err != nil {
return nil, err
}
out := make([]types.Resource, 0, len(spaces))
for _, s := range spaces {
metadata := map[string]interface{}{"space_key": s.Key}
if cfg.cloud() {
// Cloud cannot enumerate a space's top-level folders, so pages under
// them are unselectable in the picker; the frontend surfaces this
// limitation when the space is expanded.
metadata["hierarchy_limitation"] = "cloud_top_level_containers"
}
out = append(out, types.Resource{
ExternalID: s.ID, Name: s.Name, Type: "space",
URL: client.resourceURL(s.Links.WebUI), HasChildren: true,
Metadata: metadata,
})
}
return out, nil
}
ref, err := parseResourceID(parentID)
if err != nil {
return nil, err
}
spaces, err := c.pickerSpaces(ctx, client, cfg)
if err != nil {
return nil, err
}
var selected space
found := false
for _, s := range spaces {
if s.ID == ref.SpaceID {
selected, found = s, true
break
}
}
if !found {
return nil, fmt.Errorf("confluence space %s is unavailable", ref.SpaceID)
}
var pages []page
if ref.Kind == resourceSpace {
pages, err = client.topLevelPages(ctx, selected)
} else {
if ref.Kind == resourcePage {
if err := client.validatePageOwnership(ctx, selected, ref.PageID); err != nil {
return nil, fmt.Errorf("validate Confluence page %s: %w", ref.PageID, err)
}
}
pages, err = client.directChildPages(ctx, selected, ref.PageID)
}
if err != nil {
return nil, fmt.Errorf("list Confluence resources under %s: %w", parentID, err)
}
out := make([]types.Resource, 0, len(pages))
for _, p := range pages {
out = append(out, pageResource(client, selected, parentID, p))
}
return out, nil
}
func pageResource(client *client, s space, parentID string, p page) types.Resource {
return types.Resource{
ExternalID: makePageResourceID(s.ID, p.ID), ParentID: parentID, Name: p.Title, Type: "page",
URL: client.resourceURL(p.Links.WebUI), HasChildren: true,
Metadata: map[string]interface{}{"space_id": s.ID, "page_id": p.ID},
}
}
// ResolveResourceAncestors expands only the O(depth) path required to reveal
// persisted page selections in the lazy picker. Resolution is best effort:
// a deleted page, an unavailable space, or a malformed id skips just its own
// reveal path instead of failing every saved selection; the sync path keeps
// its strict validation in buildSyncPlan.
func (c *Connector) ResolveResourceAncestors(
ctx context.Context,
ds *types.DataSourceConfig,
resourceIDs []string,
) ([]string, error) {
client, _, err := c.configured(ds)
if err != nil {
return nil, err
@ -67,25 +205,47 @@ func (c *Connector) ListResources(
if err != nil {
return nil, err
}
out := make([]types.Resource, 0, len(spaces))
byID := make(map[string]space, len(spaces))
for _, s := range spaces {
out = append(out, types.Resource{
ExternalID: s.ID,
Name: s.Name,
Type: "space",
URL: client.resourceURL(s.Links.WebUI),
HasChildren: false,
Metadata: map[string]interface{}{"space_key": s.Key},
})
byID[s.ID] = s
}
seen, out := map[string]bool{}, []string{}
add := func(id string) {
if id != "" && !seen[id] {
seen[id] = true
out = append(out, id)
}
}
for _, id := range resourceIDs {
ref, err := parseResourceID(id)
if err != nil {
logger.Warnf(ctx, "skip malformed Confluence resource %q during ancestor resolution: %v", id, err)
continue
}
if ref.Kind == resourceSpace {
continue
}
s, ok := byID[ref.SpaceID]
if !ok {
logger.Warnf(ctx, "skip Confluence resource %s: space %s is unavailable", id, ref.SpaceID)
continue
}
ancestors, err := client.pageAncestors(ctx, s, ref.PageID)
if err != nil {
logger.Warnf(ctx, "skip Confluence resource %s during ancestor resolution: %v", id, err)
continue
}
add(s.ID)
for _, ancestor := range ancestors {
if ancestor.Kind != "" && ancestor.Kind != "page" {
continue
}
add(makePageResourceID(s.ID, ancestor.ID))
}
}
return out, nil
}
// ResolveResourceAncestors is a no-op: Confluence spaces are a flat list.
func (*Connector) ResolveResourceAncestors(context.Context, *types.DataSourceConfig, []string) ([]string, error) {
return nil, nil
}
// FetchAll syncs every selected space. Deletion reconciliation needs FetchStream.
func (c *Connector) FetchAll(
ctx context.Context, ds *types.DataSourceConfig, resourceIDs []string,
@ -158,50 +318,89 @@ func (c *Connector) fetchStream(
if err != nil {
return nil, err
}
spaces, err := client.spaces(ctx)
plan, err := c.buildSyncPlan(ctx, client, ds.ResourceIDs)
if err != nil {
return nil, fmt.Errorf("list Confluence spaces: %w", err)
return nil, err
}
byID := make(map[string]space, len(spaces))
for _, s := range spaces {
byID[s.ID] = s
currentRoots := make(map[string]bool, len(plan.Roots))
for _, root := range plan.Roots {
currentRoots[root.ResourceID] = true
}
baseline, next := prepareSyncCursors(old, forceFull)
for _, resourceID := range ds.ResourceIDs {
s, found := byID[resourceID]
if !found {
return nil, fmt.Errorf(
"selected Confluence space %s is unavailable; refusing destructive reconciliation",
resourceID,
)
// A prior successful body is reusable when ownership moves to a different
// selected root and this run cannot fetch the new body. Keep that version in
// the new root rather than dropping it during ownership reconciliation.
priorVersionByPage := make(map[string]string)
for _, pages := range baseline.SpacePages {
for id, version := range pages {
if _, exists := priorVersionByPage[id]; !exists {
priorVersionByPage[id] = version
}
}
}
// A page may be returned by multiple overlapping roots (for example during
// a configuration change). Fetch it once, while retaining a cursor entry
// per scope root so existing cursor JSON remains compatible.
seenPages := make(map[string]struct{})
incompleteRoots := make(map[string]bool)
incompleteSpaces := make([]space, 0)
for _, root := range plan.Roots {
resourceID, s := root.ResourceID, root.Space
var pages []page
complete := true
if root.Kind == syncWholeSpace {
pages, complete, err = client.pages(ctx, s)
} else {
pages, err = client.pageSubtree(ctx, s, root.PageID)
}
pages, complete, err := client.pages(ctx, s)
if err != nil {
return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)
return nil, fmt.Errorf("list pages in Confluence scope %s: %w", resourceID, err)
}
priorPages := baseline.SpacePages[resourceID]
if !complete {
logger.Warnf(ctx,
"[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
"listed %d pages, checking unlisted ones one by one before deleting",
s.Key, maxEmptyServerPages, len(pages))
logger.Warnf(
ctx,
"[Confluence] space %s listing was cut short; checking unlisted pages before deleting",
s.Key,
)
incompleteRoots[resourceID] = true
incompleteSpaces = append(incompleteSpaces, s)
}
priorPages, hadBaseline := baseline.SpacePages[resourceID]
if next.SpacePages[resourceID] == nil {
next.SpacePages[resourceID] = map[string]string{}
}
seen := make(map[string]struct{}, len(pages))
if complete && len(pages) == 0 && len(priorPages) > 0 {
return nil, fmt.Errorf(
"refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",
s.Key, len(priorPages),
)
}
for _, summary := range pages {
if err := ctx.Err(); err != nil {
return nil, err
}
seen[summary.ID] = struct{}{}
version := pageVersion(summary)
if forceFull {
if next.SpacePages[resourceID][summary.ID] == version {
if _, duplicate := seenPages[summary.ID]; duplicate {
// The first scope already owns the emitted document; preserve this
// root's version record so a later selection change stays incremental.
if !forceFull {
if version, ok := priorPages[summary.ID]; ok {
next.SpacePages[resourceID][summary.ID] = version
}
}
continue
}
seenPages[summary.ID] = struct{}{}
version, versionKnown := pageVersion(summary)
// An unknown version cannot prove a page unchanged, so it must
// always refresh the body instead of skipping on a stable "t:".
if versionKnown {
if forceFull {
if next.SpacePages[resourceID][summary.ID] == version {
continue
}
} else if priorPages[summary.ID] == version {
continue
}
} else if priorPages[summary.ID] == version {
continue
}
full, err := client.body(ctx, summary.ID)
if err != nil {
@ -211,13 +410,23 @@ func (c *Connector) fetchStream(
if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil {
return nil, emitErr
}
if !forceFull {
if prior, ok := priorVersionByPage[summary.ID]; ok {
next.SpacePages[resourceID][summary.ID] = prior
}
}
continue
}
item, err := markdownItem(client, resourceID, summary, full)
item, err := markdownItem(ctx, client, resourceID, summary, full)
if err != nil {
if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil {
return nil, emitErr
}
if !forceFull {
if prior, ok := priorVersionByPage[summary.ID]; ok {
next.SpacePages[resourceID][summary.ID] = prior
}
}
continue
}
if err := h.Emit(ctx, item); err != nil {
@ -228,101 +437,119 @@ func (c *Connector) fetchStream(
return nil, err
}
}
if !complete {
if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil {
return nil, err
}
} else if hadBaseline {
if len(pages) == 0 && len(priorPages) > 0 {
return nil, fmt.Errorf(
"refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",
s.Key, len(priorPages),
)
}
missing := 0
for id := range priorPages {
if _, exists := seen[id]; !exists {
missing++
}
}
if missing >= 20 && missing*100 >= len(priorPages)*80 {
return nil, fmt.Errorf(
"refusing Confluence mirror deletion in %s: %d/%d pages disappeared",
s.Key, missing, len(priorPages),
)
}
for id := range priorPages {
if _, exists := seen[id]; exists {
continue
}
if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil {
return nil, err
}
}
// Reconcile previous owners only after every current scope has been enumerated.
// A page may move between roots, so a missing page must be checked against every
// scope whose listing was cut short before a tombstone can be emitted.
previousPages := make(map[string]string)
for owner, pages := range baseline.SpacePages {
for id := range pages {
if _, exists := previousPages[id]; !exists {
previousPages[id] = owner
}
}
}
missing := make([]string, 0)
probeCount := 0
for id := range previousPages {
if _, present := seenPages[id]; present {
continue
}
owner := previousPages[id]
for _, s := range incompleteSpaces {
if probeCount >= maxDeletionProbes {
return nil, fmt.Errorf(
"Confluence deletion reconciliation deferred: more than %d pages need verification",
maxDeletionProbes,
)
}
probeCount++
present, err := client.pageInSpace(ctx, id, s.Key)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, fmt.Errorf(
"Confluence deletion reconciliation deferred: could not verify page %s in space %s: %w",
id, s.Key, err,
)
}
if present {
version := baseline.SpacePages[owner][id]
if next.SpacePages[s.ID] == nil {
next.SpacePages[s.ID] = make(map[string]string)
}
next.SpacePages[s.ID][id] = version
seenPages[id] = struct{}{}
break
}
}
if _, present := seenPages[id]; present {
continue
}
missing = append(missing, id)
}
// Protect still-selected roots from a suspiciously large shrink. Removed
// roots are an explicit scope change, so their out-of-scope pages can be
// tombstoned once every current scope has been enumerated safely.
owners := make([]string, 0, len(baseline.SpacePages))
for owner := range baseline.SpacePages {
owners = append(owners, owner)
}
sort.Strings(owners)
for _, owner := range owners {
// A deliberate scope change may remove most of an old root. The guard is
// only for unexpected shrinkage of a still-selected, completely listed root.
if !currentRoots[owner] || incompleteRoots[owner] {
continue
}
pages := baseline.SpacePages[owner]
missingInRoot := 0
for id := range pages {
if _, present := seenPages[id]; !present {
missingInRoot++
}
}
if len(pages) > 0 && missingInRoot >= 20 && missingInRoot*100 >= len(pages)*80 {
return nil, fmt.Errorf(
"refusing Confluence mirror deletion: %d/%d previously synced pages would be removed from scope %s",
missingInRoot, len(pages), owner,
)
}
}
sort.Strings(missing)
for _, id := range missing {
deleted := types.FetchedItem{
ExternalID: id, IsDeleted: true, SourceResourceID: previousPages[id],
Metadata: map[string]string{"channel": types.ChannelConfluence},
}
if err := h.Emit(ctx, deleted); err != nil {
return nil, err
}
// Drop the tombstoned page before checkpointing so the cursor records
// the deletion itself; an interrupted run resumes with the remaining
// candidates instead of re-emitting completed tombstones.
removePageOwnership(&next, id)
if err := h.Checkpoint(ctx, next.syncCursor()); err != nil {
return nil, err
}
}
// Commit exactly the current ownership roots only after reconciliation. This
// removes cancelled scopes while keeping the on-disk cursor field compatible.
committedRoots := make(map[string]map[string]string, len(plan.Roots))
for _, root := range plan.Roots {
if pages := next.SpacePages[root.ResourceID]; pages != nil {
committedRoots[root.ResourceID] = pages
} else {
committedRoots[root.ResourceID] = map[string]string{}
}
}
next.SpacePages = committedRoots
next.FullSync = false
next.FullSyncBaseline = nil
return next.syncCursor(), nil
}
// reconcileUnlisted handles the pages a cut-short listing did not return. The
// listing proves nothing about them, so each one is looked up on its own and
// deleted only when Confluence confirms it is gone from the space. Pages that
// are still there, could not be checked, or exceed the per-run probe budget
// stay tracked with their prior version and are checked again next run.
func reconcileUnlisted(
ctx context.Context,
c *client,
h datasource.StreamHandler,
s space,
resourceID string,
priorPages map[string]string,
seen map[string]struct{},
next *cursor,
) error {
probed := 0
for id, version := range priorPages {
if _, exists := seen[id]; exists {
continue
}
if probed >= maxDeletionProbes {
next.SpacePages[resourceID][id] = version
continue
}
probed++
present, err := c.pageInSpace(ctx, id, s.Key)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err)
next.SpacePages[resourceID][id] = version
continue
}
if present {
next.SpacePages[resourceID][id] = version
continue
}
if err := emitDeleted(ctx, h, next, resourceID, id); err != nil {
return err
}
}
return nil
}
func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error {
deleted := types.FetchedItem{
ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,
Metadata: map[string]string{"channel": types.ChannelConfluence},
}
if err := h.Emit(ctx, deleted); err != nil {
return err
}
delete(next.SpacePages[resourceID], id)
return h.Checkpoint(ctx, next.syncCursor())
}
func failedPageItem(resourceID string, summary page, err error) types.FetchedItem {
code, reason := classifyConfluenceError(err)
return types.FetchedItem{
@ -361,8 +588,18 @@ func classifyConfluenceError(err error) (code, reason string) {
return "confluence_sync_failed", "Confluence page could not be synced; see server logs"
}
func markdownItem(client *client, resourceID string, summary page, full pageBody) (types.FetchedItem, error) {
func markdownItem(
ctx context.Context,
client *client,
resourceID string,
summary page,
full pageBody,
) (types.FetchedItem, error) {
html := full.Body.View.Value
// Inline same-origin private Confluence images as data URIs so the generic
// docparser image pipeline can persist them and rewrite to resource:// URLs.
// Best-effort: a failed image keeps its original src and never fails the page.
html = newAssetResolver(client).Resolve(ctx, html)
markdown, err := htmltomd.ConvertString(html)
if err != nil {
return types.FetchedItem{}, err

View File

@ -0,0 +1,43 @@
package confluence
import (
"fmt"
"strings"
)
type resourceKind uint8
const (
resourceSpace resourceKind = iota
resourcePage
)
// resourceRef is the canonical, persisted representation of a selection.
// Space IDs deliberately remain unprefixed for backwards compatibility.
type resourceRef struct {
Kind resourceKind
SpaceID string
PageID string
}
func makePageResourceID(spaceID, pageID string) string {
return "page:" + spaceID + ":" + pageID
}
func parseResourceID(id string) (resourceRef, error) {
id = strings.TrimSpace(id)
if id == "" {
return resourceRef{}, fmt.Errorf("invalid empty Confluence resource ID")
}
if !strings.HasPrefix(id, "page:") {
if strings.Contains(id, ":") {
return resourceRef{}, fmt.Errorf("invalid Confluence space resource ID %q", id)
}
return resourceRef{Kind: resourceSpace, SpaceID: id}, nil
}
parts := strings.Split(id, ":")
if len(parts) != 3 || parts[0] != "page" || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" {
return resourceRef{}, fmt.Errorf("invalid Confluence page resource ID %q", id)
}
return resourceRef{Kind: resourcePage, SpaceID: parts[1], PageID: parts[2]}, nil
}

View File

@ -0,0 +1,23 @@
package confluence
import "testing"
func TestParseResourceID(t *testing.T) {
space, err := parseResourceID("86802433")
if err != nil || space.Kind != resourceSpace || space.SpaceID != "86802433" {
t.Fatalf("space = %#v, %v", space, err)
}
page, err := parseResourceID("page:86802433:123456789")
if err != nil || page.Kind != resourcePage || page.SpaceID != "86802433" || page.PageID != "123456789" {
t.Fatalf("page = %#v, %v", page, err)
}
invalid := []string{
"", "page:", "page:100", "page::200", "page:100:", "page:100:200:300",
"node:86802433:folder-1", "space:100",
}
for _, id := range invalid {
if _, err := parseResourceID(id); err == nil {
t.Errorf("parseResourceID(%q) succeeded", id)
}
}
}

View File

@ -0,0 +1,112 @@
package confluence
import (
"context"
"fmt"
"sort"
)
type syncRootKind uint8
const (
syncWholeSpace syncRootKind = iota
syncPageSubtree
)
type syncRoot struct {
ResourceID string
SpaceID string
Space space
PageID string
Kind syncRootKind
}
type syncPlan struct{ Roots []syncRoot }
// buildSyncPlan validates selections before any destructive reconciliation and
// removes scopes which are already covered by a selected ancestor.
func (c *Connector) buildSyncPlan(ctx context.Context, client *client, resourceIDs []string) (*syncPlan, error) {
spaces, err := client.spaces(ctx)
if err != nil {
return nil, fmt.Errorf("list Confluence spaces: %w", err)
}
byID := make(map[string]space, len(spaces))
for _, s := range spaces {
byID[s.ID] = s
}
plan := &syncPlan{}
wholeSpaces := make(map[string]bool)
pageRoots := make(map[string]syncRoot)
ancestorPaths := make(map[string][]page)
for _, resourceID := range resourceIDs {
ref, err := parseResourceID(resourceID)
if err != nil {
return nil, err
}
s, ok := byID[ref.SpaceID]
if !ok {
return nil, fmt.Errorf(
"selected Confluence space %s is unavailable; refusing destructive reconciliation",
ref.SpaceID,
)
}
if ref.Kind == resourceSpace {
wholeSpaces[ref.SpaceID] = true
continue
}
// pageAncestors fetches the page itself on Server/DC and Cloud; that is
// also the authoritative check that a client did not forge its space ID.
ancestors, err := client.pageAncestors(ctx, s, ref.PageID)
if err != nil {
return nil, fmt.Errorf("validate selected Confluence page %s: %w", ref.PageID, err)
}
pageRoots[resourceID] = syncRoot{
ResourceID: resourceID,
SpaceID: ref.SpaceID,
Space: s,
PageID: ref.PageID,
Kind: syncPageSubtree,
}
ancestorPaths[resourceID] = ancestors
for _, ancestor := range ancestors {
if ancestor.ID == ref.PageID {
return nil, fmt.Errorf("invalid cyclic Confluence page ancestry for %s", ref.PageID)
}
}
}
spaceIDs := make([]string, 0, len(wholeSpaces))
for id := range wholeSpaces {
spaceIDs = append(spaceIDs, id)
}
sort.Strings(spaceIDs)
for _, id := range spaceIDs {
plan.Roots = append(plan.Roots, syncRoot{ResourceID: id, SpaceID: id, Space: byID[id], Kind: syncWholeSpace})
}
pageIDs := make([]string, 0, len(pageRoots))
for id := range pageRoots {
pageIDs = append(pageIDs, id)
}
sort.Strings(pageIDs)
for _, id := range pageIDs {
root := pageRoots[id]
if wholeSpaces[root.SpaceID] {
continue
}
ancestors := ancestorPaths[id]
covered := false
for _, ancestor := range ancestors {
if ancestor.Kind != "" && ancestor.Kind != "page" {
continue
}
if _, ok := pageRoots[makePageResourceID(root.SpaceID, ancestor.ID)]; ok {
covered = true
break
}
}
if !covered {
plan.Roots = append(plan.Roots, pageRoots[id])
}
}
return plan, nil
}

View File

@ -95,10 +95,11 @@ func isAtlassianCloudHost(host string) bool {
}
type space struct {
ID string `json:"id"`
Key string `json:"key"`
Name string `json:"name"`
Links struct {
ID string `json:"id"`
Key string `json:"key"`
Name string `json:"name"`
HomepageID string `json:"homepageId"`
Links struct {
WebUI string `json:"webui"`
} `json:"_links"`
}
@ -125,10 +126,13 @@ type serverSpaceList struct {
}
type page struct {
ID string `json:"id"`
Title string `json:"title"`
Status string `json:"status"`
Space struct {
ID string `json:"id"`
Title string `json:"title"`
Status string `json:"status"`
Kind string `json:"-"`
SpaceID string `json:"spaceId"`
Ancestors []page `json:"ancestors"`
Space struct {
Key string `json:"key"`
Name string `json:"name"`
} `json:"space"`
@ -189,6 +193,27 @@ type cloudPage struct {
} `json:"_links"`
}
// cloudHierarchyItem is shared by Cloud direct-children, descendants, and
// ancestors. Non-page items are intentionally retained by the API model so we
// can filter them explicitly at the navigation/import boundary.
type cloudHierarchyItem struct {
ID string `json:"id"`
Type string `json:"type"`
Title string `json:"title"`
SpaceID string `json:"spaceId"`
Status string `json:"status"`
Links struct {
WebUI string `json:"webui"`
} `json:"_links"`
}
type cloudHierarchyList struct {
Results []cloudHierarchyItem `json:"results"`
Links struct {
Next string `json:"next"`
} `json:"_links"`
}
type cloudPageList struct {
Results []cloudPage `json:"results"`
Links struct {
@ -278,14 +303,37 @@ func prepareSyncCursors(old *types.SyncCursor, forceFull bool) (baseline, next c
return baseline, next
}
func pageVersion(p page) string {
// removePageOwnership drops a page from every root ownership in the cursor.
// It runs after each tombstone emit so the checkpoint mirrors the deletion;
// otherwise the page would linger as a ghost, be re-tombstoned on every later
// run, and eventually trip the mass-deletion guard. During a full sync the
// deletion baseline lives in FullSyncBaseline, not SpacePages (which is being
// rebuilt from scratch), so both maps must record the completed tombstone for
// a retry to resume with the remaining candidates only.
func removePageOwnership(c *cursor, pageID string) {
removeFromPageMap(c.SpacePages, pageID)
if c.FullSync {
removeFromPageMap(c.FullSyncBaseline, pageID)
}
}
func removeFromPageMap(m map[string]map[string]string, pageID string) {
for _, pages := range m {
delete(pages, pageID)
}
}
func pageVersion(p page) (string, bool) {
if p.Version.Number > 0 {
return fmt.Sprintf("v:%d", p.Version.Number)
return fmt.Sprintf("v:%d", p.Version.Number), true
}
if p.Version.When != "" {
return "t:" + p.Version.When
return "t:" + p.Version.When, true
}
return "t:" + p.Version.CreatedAt
if p.Version.CreatedAt != "" {
return "t:" + p.Version.CreatedAt, true
}
return "", false
}
func pageUpdatedAt(p page) time.Time {

View File

@ -75,6 +75,34 @@ func TestPageFileNameIncludesID(t *testing.T) {
}
}
func TestPageVersionUnknown(t *testing.T) {
withNumber := page{}
withNumber.Version.Number = 7
withWhen := page{}
withWhen.Version.When = "2026-01-02T03:04:05Z"
withCreatedAt := page{}
withCreatedAt.Version.CreatedAt = "2026-01-02T03:04:05Z"
for _, tc := range []struct {
name string
in page
want string
wantKnown bool
}{
{"number", withNumber, "v:7", true},
{"when", withWhen, "t:2026-01-02T03:04:05Z", true},
{"createdAt", withCreatedAt, "t:2026-01-02T03:04:05Z", true},
{"unknown", page{}, "", false},
} {
t.Run(tc.name, func(t *testing.T) {
got, known := pageVersion(tc.in)
if got != tc.want || known != tc.wantKnown {
t.Fatalf("pageVersion() = (%q, %v); want (%q, %v)", got, known, tc.want, tc.wantKnown)
}
})
}
}
func TestPrepareSyncCursorsPreservesFullSyncBaseline(t *testing.T) {
old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
baseline, next := prepareSyncCursors(old, true)