diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go index 05c9d977..ec156e38 100644 --- a/internal/datasource/connector/confluence/client.go +++ b/internal/datasource/connector/confluence/client.go @@ -3,6 +3,7 @@ package confluence import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -26,6 +27,9 @@ const ( // next past the end of a space (#3596); permission filtering can also // empty one page mid-listing, so a single empty page is not the end. maxEmptyServerPages = 3 + // maxDeletionProbes caps how many unlisted pages one sync looks up + // individually after a cut-short listing; the rest wait for the next run. + maxDeletionProbes = 200 ) type apiError struct { @@ -367,6 +371,31 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo return all, complete, err } +// pageInSpace reports whether a Server page is still a current page of the +// space. Confluence answers 404 for a page that was deleted or moved to the +// trash; a page that is no longer current or now belongs to another space is +// gone from this one too, matching what a complete listing would omit. +func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) { + var result struct { + Status string `json:"status"` + Space struct { + Key string `json:"key"` + } `json:"space"` + } + err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=space", &result) + var apiErr *apiError + if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound { + return false, nil + } + if err != nil { + return false, err + } + if result.Status != "" && result.Status != "current" { + return false, nil + } + return result.Space.Key == "" || result.Space.Key == spaceKey, nil +} + func (c *client) body(ctx context.Context, id string) (pageBody, error) { if c.cfg.cloud() { var result pageBody diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go index a0407e87..1a7df281 100644 --- a/internal/datasource/connector/confluence/client_pages_test.go +++ b/internal/datasource/connector/confluence/client_pages_test.go @@ -177,3 +177,34 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) { t.Fatalf("depth query = %#v", depths) } } + +func TestPageInSpace(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("expand") != "space" { + t.Errorf("expand = %q, want space", r.URL.Query().Get("expand")) + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/rest/api/content/current": + _, _ = w.Write([]byte(`{"id": "current", "status": "current", "space": {"key": "ENG"}}`)) + case "/rest/api/content/trashed": + _, _ = w.Write([]byte(`{"id": "trashed", "status": "trashed", "space": {"key": "ENG"}}`)) + case "/rest/api/content/moved": + _, _ = w.Write([]byte(`{"id": "moved", "status": "current", "space": {"key": "OPS"}}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + + c := &client{cfg: config{baseURL: server.URL}, http: server.Client()} + for id, want := range map[string]bool{"current": true, "trashed": false, "moved": false, "deleted": false} { + got, err := c.pageInSpace(context.Background(), id, "ENG") + if err != nil { + t.Fatalf("%s: %v", id, err) + } + if got != want { + t.Errorf("pageInSpace(%s) = %v, want %v", id, got, want) + } + } +} diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go index b753484a..fe2a6f5c 100644 --- a/internal/datasource/connector/confluence/connector.go +++ b/internal/datasource/connector/confluence/connector.go @@ -182,7 +182,7 @@ func (c *Connector) fetchStream( if !complete { logger.Warnf(ctx, "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+ - "imported %d pages and skipped deletion reconciliation", + "listed %d pages, checking unlisted ones one by one before deleting", s.Key, maxEmptyServerPages, len(pages)) } priorPages, hadBaseline := baseline.SpacePages[resourceID] @@ -229,13 +229,8 @@ func (c *Connector) fetchStream( } } if !complete { - // A cut-short listing proves nothing about the pages it did not - // reach: keep tracking them so a later complete listing can still - // reconcile them, and delete nothing now. - for id, version := range priorPages { - if _, exists := seen[id]; !exists { - next.SpacePages[resourceID][id] = version - } + if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil { + return nil, err } } else if hadBaseline { if len(pages) == 0 && len(priorPages) > 0 { @@ -260,15 +255,7 @@ func (c *Connector) fetchStream( if _, exists := seen[id]; exists { continue } - deleted := types.FetchedItem{ - ExternalID: id, IsDeleted: true, SourceResourceID: resourceID, - Metadata: map[string]string{"channel": types.ChannelConfluence}, - } - if err := h.Emit(ctx, deleted); err != nil { - return nil, err - } - delete(next.SpacePages[resourceID], id) - if err := h.Checkpoint(ctx, next.syncCursor()); err != nil { + if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil { return nil, err } } @@ -279,6 +266,63 @@ func (c *Connector) fetchStream( return next.syncCursor(), nil } +// reconcileUnlisted handles the pages a cut-short listing did not return. The +// listing proves nothing about them, so each one is looked up on its own and +// deleted only when Confluence confirms it is gone from the space. Pages that +// are still there, could not be checked, or exceed the per-run probe budget +// stay tracked with their prior version and are checked again next run. +func reconcileUnlisted( + ctx context.Context, + c *client, + h datasource.StreamHandler, + s space, + resourceID string, + priorPages map[string]string, + seen map[string]struct{}, + next *cursor, +) error { + probed := 0 + for id, version := range priorPages { + if _, exists := seen[id]; exists { + continue + } + if probed >= maxDeletionProbes { + next.SpacePages[resourceID][id] = version + continue + } + probed++ + present, err := c.pageInSpace(ctx, id, s.Key) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err) + next.SpacePages[resourceID][id] = version + continue + } + if present { + next.SpacePages[resourceID][id] = version + continue + } + if err := emitDeleted(ctx, h, next, resourceID, id); err != nil { + return err + } + } + return nil +} + +func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error { + deleted := types.FetchedItem{ + ExternalID: id, IsDeleted: true, SourceResourceID: resourceID, + Metadata: map[string]string{"channel": types.ChannelConfluence}, + } + if err := h.Emit(ctx, deleted); err != nil { + return err + } + delete(next.SpacePages[resourceID], id) + return h.Checkpoint(ctx, next.syncCursor()) +} + func failedPageItem(resourceID string, summary page, err error) types.FetchedItem { code, reason := classifyConfluenceError(err) return types.FetchedItem{ diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go index 07a024b0..2f38c787 100644 --- a/internal/datasource/connector/confluence/connector_stream_test.go +++ b/internal/datasource/connector/confluence/connector_stream_test.go @@ -34,6 +34,16 @@ type streamAPI struct { // endlessNext mimics Server/Data Center builds that keep linking to a // next page after the last one, returning empty results forever (#3596). endlessNext bool + // probe answers the per-page existence check (GET content/{id}?expand=space) + // for pages that are not in the listing: an HTTP status, or 200 with the + // space key the page now belongs to. + probe map[string]probeReply + probeCalls int +} + +type probeReply struct { + status int + spaceKey string } func (a *streamAPI) jsonResponse(value any) (*http.Response, error) { @@ -84,6 +94,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) { "results": results, "_links": map[string]any{"next": next}, }) + case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space": + a.probeCalls++ + id := strings.TrimPrefix(path, "/wiki/rest/api/content/") + reply, ok := a.probe[id] + if !ok { + return nil, errors.New("unexpected existence check for page " + id) + } + if reply.status != http.StatusOK { + return &http.Response{ + StatusCode: reply.status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("{}")), + }, nil + } + return a.jsonResponse(map[string]any{ + "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}, + }) case strings.HasPrefix(path, "/wiki/rest/api/content/"): if req.URL.Query().Get("expand") != "body.view,version,space" { return nil, errors.New("page body did not request rendered body.view") @@ -403,12 +430,23 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) { } } -func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) { +func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) { for _, full := range []bool{false, true} { t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) { - api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}} + api := &streamAPI{ + endlessNext: true, + pages: []streamPage{{id: "p1", title: "Page", version: 2}}, + probe: map[string]probeReply{ + "gone": {status: http.StatusNotFound}, + "moved": {status: http.StatusOK, spaceKey: "OPS"}, + "kept": {status: http.StatusOK, spaceKey: "ENG"}, + "denied": {status: http.StatusForbidden}, + }, + } h := &captureHandler{} - old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}) + old := streamCursor(map[string]string{ + "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1", + }) connector := newStreamConnector(api) fetch := connector.FetchStream if full { @@ -421,17 +459,27 @@ func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) if api.listCalls != 1+maxEmptyServerPages { t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages) } + if api.probeCalls != 4 { + t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls) + } + deleted := map[string]bool{} for _, item := range h.items { if item.IsDeleted { - t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID) + deleted[item.ExternalID] = true } } - if len(h.items) != 1 || h.items[0].ExternalID != "p1" { - t.Fatalf("items = %#v, want the changed page p1 only", h.items) + if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] { + t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted) } pages := decodeCursor(got).SpacePages["1"] - if pages["p1"] != "v:2" || pages["p2"] != "v:1" { - t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages) + want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"} + if len(pages) != len(want) { + t.Fatalf("cursor pages = %#v, want %#v", pages, want) + } + for id, version := range want { + if pages[id] != version { + t.Fatalf("cursor pages = %#v, want %#v", pages, want) + } } }) }