diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 9af1e47e..61ec2d27 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -1257,6 +1257,9 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource if !item.UpdatedAt.IsZero() { metadata["source_updated_at"] = item.UpdatedAt.UTC().Format(time.RFC3339) } + if !item.CreatedAt.IsZero() { + metadata["source_created_at"] = item.CreatedAt.UTC().Format(time.RFC3339) + } for k, v := range item.Metadata { metadata[k] = v } diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.go index 445607c8..4a64723a 100644 --- a/internal/application/service/datasource_service_test.go +++ b/internal/application/service/datasource_service_test.go @@ -511,14 +511,17 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) { repo := &deletionLookupKnowledgeRepo{} ks := &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-1"}} svc := &DataSourceService{knowledgeService: ks} + created := time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC) _, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{ ExternalID: "url:1", URL: "https://example.com/doc", UpdatedAt: edited, + CreatedAt: created, }, nil) require.NoError(t, err) require.Len(t, repo.metadataUpdates, 1) assert.Equal(t, "2023-04-04T22:07:08Z", repo.metadataUpdates[0]["source_updated_at"]) + assert.Equal(t, "2021-01-02T03:04:05Z", repo.metadataUpdates[0]["source_created_at"]) repo = &deletionLookupKnowledgeRepo{} ks = &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-2"}} @@ -531,6 +534,8 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) { require.Len(t, repo.metadataUpdates, 1) _, present := repo.metadataUpdates[0]["source_updated_at"] assert.False(t, present) + _, present = repo.metadataUpdates[0]["source_created_at"] + assert.False(t, present) } func TestProcessSync_SyncDeletionsDeletesMatchingKnowledge(t *testing.T) { diff --git a/internal/datasource/connector/feishu/core/shared.go b/internal/datasource/connector/feishu/core/shared.go index 5900ecf0..fb6cbb00 100644 --- a/internal/datasource/connector/feishu/core/shared.go +++ b/internal/datasource/connector/feishu/core/shared.go @@ -290,11 +290,13 @@ type DocxFetchInput struct { // WeKnora external_id: wiki=node.NodeToken, drive=file.Token DocToken string // Feishu docx document token - ObjToken string - Title string - URL string - ResourceID string - EditTime time.Time + ObjToken string + Title string + URL string + ResourceID string + EditTime time.Time + // CreateTime is the document creation time in Feishu; zero when unknown. + CreateTime time.Time BaseMeta map[string]string MultimodalEnabled bool } @@ -364,6 +366,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput) FileName: SanitizeFileName(in.Title) + ".md", URL: in.URL, UpdatedAt: in.EditTime, + CreatedAt: in.CreateTime, SourceResourceID: in.ResourceID, Metadata: in.BaseMeta, ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync @@ -414,6 +417,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput) FileName: SanitizeFileName(a.Name), URL: in.URL, UpdatedAt: in.EditTime, + CreatedAt: in.CreateTime, SourceResourceID: in.ResourceID, Metadata: childMeta(), }) @@ -463,6 +467,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput) FileName: "image-" + b.Image.Token + ext, URL: in.URL, UpdatedAt: in.EditTime, + CreatedAt: in.CreateTime, SourceResourceID: in.ResourceID, Metadata: imgMeta(), }) @@ -496,6 +501,7 @@ func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput) FileName: fileName, URL: in.URL, UpdatedAt: in.EditTime, + CreatedAt: in.CreateTime, SourceResourceID: in.ResourceID, Metadata: in.BaseMeta, }, nil diff --git a/internal/datasource/connector/feishu/drive/connector.go b/internal/datasource/connector/feishu/drive/connector.go index 1e2d2432..41acdb5a 100644 --- a/internal/datasource/connector/feishu/drive/connector.go +++ b/internal/datasource/connector/feishu/drive/connector.go @@ -363,6 +363,7 @@ func fetchDriveFileContent( } editTime := core.ParseFeishuTimestamp(file.ModifiedTime) + createTime := core.ParseFeishuTimestamp(file.CreatedTime) // Channel marks the knowledge "source" label. Drive uses its own channel // (feishu_drive / lark_drive) so Drive docs show "飞书云盘" / "Lark 云盘" // distinct from the wiki connector's "飞书". @@ -387,6 +388,7 @@ func fetchDriveFileContent( URL: file.URL, ResourceID: resourceID, EditTime: editTime, + CreateTime: createTime, BaseMeta: baseMeta, MultimodalEnabled: multimodalEnabled, }) @@ -412,6 +414,7 @@ func fetchDriveFileContent( FileName: fileName, URL: file.URL, UpdatedAt: editTime, + CreatedAt: createTime, SourceResourceID: resourceID, Metadata: baseMeta, }}, nil @@ -435,6 +438,7 @@ func fetchDriveFileContent( FileName: fileName, URL: file.URL, UpdatedAt: editTime, + CreatedAt: createTime, SourceResourceID: resourceID, Metadata: baseMeta, }}, nil diff --git a/internal/datasource/connector/feishu/wiki/connector.go b/internal/datasource/connector/feishu/wiki/connector.go index 1fc35944..761c2705 100644 --- a/internal/datasource/connector/feishu/wiki/connector.go +++ b/internal/datasource/connector/feishu/wiki/connector.go @@ -243,8 +243,9 @@ func (o wikiOps) Title(n core.WikiNode) string { return n.Title } func (o wikiOps) ObjType(n core.WikiNode) string { return n.ObjType } // EditTime is the change-detection timestamp: ObjEditTime (document content) -// with a NodeEditTime fallback for nodes that lack obj_edit_time. This drives -// the cursor comparison, NOT FetchedItem.UpdatedAt (which uses NodeEditTime). +// with a NodeEditTime fallback for nodes that lack obj_edit_time. It drives the +// cursor comparison and, parsed, FetchedItem.UpdatedAt (see contentEditTime), +// so the persisted source_updated_at tracks content edits, not node moves. func (o wikiOps) EditTime(n core.WikiNode) string { if n.ObjEditTime != "" { return n.ObjEditTime @@ -323,7 +324,8 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo return nil, nil } - editTime := core.ParseFeishuTimestamp(node.NodeEditTime) + editTime := contentEditTime(node) + createTime := contentCreateTime(node) baseMeta := map[string]string{ "obj_token": node.ObjToken, "obj_type": node.ObjType, @@ -343,6 +345,7 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo URL: region.WikiURL(node.NodeToken), ResourceID: resourceID, EditTime: editTime, + CreateTime: createTime, BaseMeta: baseMeta, MultimodalEnabled: multimodalEnabled, }) @@ -351,12 +354,14 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo if err != nil { return nil, err } + item.CreatedAt = createTime return []*types.FetchedItem{item}, nil case "file": item, err := fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta, region) if err != nil { return nil, err } + item.CreatedAt = createTime return []*types.FetchedItem{item}, nil default: return nil, nil @@ -422,6 +427,26 @@ func fetchDriveFile(ctx context.Context, client *core.Client, node core.WikiNode }, nil } +// contentEditTime is the document's last content edit time (obj_edit_time), +// falling back to the node attribute edit time when Feishu omits it. It is +// what ingestion persists as source_updated_at; NodeEditTime alone would move +// on a rename or a move in the tree without any content change. +func contentEditTime(node core.WikiNode) time.Time { + if t := core.ParseFeishuTimestamp(node.ObjEditTime); !t.IsZero() { + return t + } + return core.ParseFeishuTimestamp(node.NodeEditTime) +} + +// contentCreateTime is the document creation time (obj_create_time), falling +// back to the node creation time. Persisted as source_created_at. +func contentCreateTime(node core.WikiNode) time.Time { + if t := core.ParseFeishuTimestamp(node.ObjCreateTime); !t.IsZero() { + return t + } + return core.ParseFeishuTimestamp(node.NodeCreateTime) +} + // --- Helper functions --- func makeWikiNodeResourceID(spaceID, nodeToken string) string { diff --git a/internal/datasource/connector/feishu/wiki/connector_test.go b/internal/datasource/connector/feishu/wiki/connector_test.go index 7d52a118..305624a9 100644 --- a/internal/datasource/connector/feishu/wiki/connector_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_test.go @@ -579,11 +579,14 @@ func TestConnectorResolveResourceAncestors(t *testing.T) { func TestFetchAll_DocxNode(t *testing.T) { nodes := []core.WikiNode{{ - NodeToken: "nt1", - ObjToken: "obj-docx-1", - ObjType: "docx", - Title: "My Document", - NodeEditTime: "1711468800", + NodeToken: "nt1", + ObjToken: "obj-docx-1", + ObjType: "docx", + Title: "My Document", + ObjCreateTime: "1700000000", + ObjEditTime: "1711000000", + NodeCreateTime: "1700000001", + NodeEditTime: "1711468800", }} ts, cfg := fakeFeishu(nodes) defer ts.Close() @@ -617,6 +620,27 @@ func TestFetchAll_DocxNode(t *testing.T) { if item.Metadata["channel"] != types.ChannelFeishu { t.Errorf("Metadata[channel] = %q", item.Metadata["channel"]) } + // Source timestamps come from the document (obj_*), not the wiki node + // attributes, so a rename or move does not look like a content edit. + if got := item.UpdatedAt.Unix(); got != 1711000000 { + t.Errorf("UpdatedAt = %d, want obj_edit_time 1711000000", got) + } + if got := item.CreatedAt.Unix(); got != 1700000000 { + t.Errorf("CreatedAt = %d, want obj_create_time 1700000000", got) + } +} + +func TestContentTimes_FallBackToNodeTimes(t *testing.T) { + n := core.WikiNode{NodeCreateTime: "1700000001", NodeEditTime: "1711468800"} + if got := contentEditTime(n).Unix(); got != 1711468800 { + t.Errorf("contentEditTime fallback = %d, want node_edit_time", got) + } + if got := contentCreateTime(n).Unix(); got != 1700000001 { + t.Errorf("contentCreateTime fallback = %d, want node_create_time", got) + } + if !contentEditTime(core.WikiNode{}).IsZero() || !contentCreateTime(core.WikiNode{}).IsZero() { + t.Errorf("missing times must stay zero, not epoch") + } } func TestFetchAll_SheetNode(t *testing.T) { diff --git a/internal/datasource/connector/gitlab/connector.go b/internal/datasource/connector/gitlab/connector.go index ec512e71..b8241954 100644 --- a/internal/datasource/connector/gitlab/connector.go +++ b/internal/datasource/connector/gitlab/connector.go @@ -404,7 +404,24 @@ func (c *Connector) item(ctx context.Context, p *project, ref, file string) (typ return types.FetchedItem{}, err } id := fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file) - return types.FetchedItem{ExternalID: id, Title: p.PathWithNamespace + "/" + file, FileName: knowledgeRelativePath(p.Name, ref, file), Content: body, ContentType: "text/plain", UpdatedAt: time.Now().UTC(), SourceResourceID: fmt.Sprint(p.ID), Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "source_type": "gitlab", "gitlab_project_id": fmt.Sprint(p.ID), "gitlab_ref": ref, "gitlab_path": file, "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file}}, nil + // UpdatedAt is intentionally left unset: the file's last commit time is not + // fetched here, and a fetch timestamp would be a fabricated source time. + return types.FetchedItem{ + ExternalID: id, + Title: p.PathWithNamespace + "/" + file, + FileName: knowledgeRelativePath(p.Name, ref, file), + Content: body, + ContentType: "text/plain", + SourceResourceID: fmt.Sprint(p.ID), + Metadata: map[string]string{ + "channel": types.ConnectorTypeGitLab, + "source_type": "gitlab", + "gitlab_project_id": fmt.Sprint(p.ID), + "gitlab_ref": ref, + "gitlab_path": file, + "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file, + }, + }, nil } // knowledgeRelativePath maps a repository file to the KB folder convention: diff --git a/internal/types/datasource.go b/internal/types/datasource.go index a1ba855c..5130b931 100644 --- a/internal/types/datasource.go +++ b/internal/types/datasource.go @@ -332,6 +332,9 @@ type FetchedItem struct { // When last modified in external system UpdatedAt time.Time `json:"updated_at"` + // When created in external system. Zero when the source does not expose it. + CreatedAt time.Time `json:"created_at"` + // Additional metadata to preserve Metadata map[string]string `json:"metadata"`