feat(datasource): persist source created time; Feishu uses content edit time for source_updated_at (#3083)

* feat(datasource): persist source document creation time and use content edit time for Feishu

Follow-up to 160c806b (Refs #3016), which started persisting the source
last-modified time as metadata.source_updated_at. Two gaps remained:

1. Creation time was never carried. FetchedItem gains CreatedAt and
   ingestItem persists it as metadata.source_created_at (RFC3339, UTC)
   with the same absent-means-unknown rule as source_updated_at.

2. The Feishu wiki connector filled FetchedItem.UpdatedAt from
   NodeEditTime, which moves when a node is renamed or dragged in the
   tree without any content change. The connector's own cursor already
   prefers ObjEditTime for exactly this reason; use the same rule for the
   persisted value (contentEditTime / contentCreateTime, node_* as
   fallback), so source_updated_at reflects document edits.

Feishu Drive populates CreatedAt from created_time. GitLab stops
fabricating UpdatedAt = time.Now(): a fetch timestamp is not a document
timestamp, and an absent value is the documented "unknown" (the commit
time is not fetched today; leaving it unset is honest until it is).

Tests: ingestItem covers both keys present/absent; the wiki fetch test
asserts obj_* times win and node_* are the fallback.

* style(gitlab): break up the FetchedItem literal (lll)

---------

Co-authored-by: xiaoyaner0201 <>
main
小妍儿 ✨ 2026-09-07 19:39:01 +08:00 committed by GitHub
parent 6a55860431
commit 061df8efdc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 101 additions and 14 deletions

View File

@ -1257,6 +1257,9 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
if !item.UpdatedAt.IsZero() {
metadata["source_updated_at"] = item.UpdatedAt.UTC().Format(time.RFC3339)
}
if !item.CreatedAt.IsZero() {
metadata["source_created_at"] = item.CreatedAt.UTC().Format(time.RFC3339)
}
for k, v := range item.Metadata {
metadata[k] = v
}

View File

@ -511,14 +511,17 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
repo := &deletionLookupKnowledgeRepo{}
ks := &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-1"}}
svc := &DataSourceService{knowledgeService: ks}
created := time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)
_, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{
ExternalID: "url:1",
URL: "https://example.com/doc",
UpdatedAt: edited,
CreatedAt: created,
}, nil)
require.NoError(t, err)
require.Len(t, repo.metadataUpdates, 1)
assert.Equal(t, "2023-04-04T22:07:08Z", repo.metadataUpdates[0]["source_updated_at"])
assert.Equal(t, "2021-01-02T03:04:05Z", repo.metadataUpdates[0]["source_created_at"])
repo = &deletionLookupKnowledgeRepo{}
ks = &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-2"}}
@ -531,6 +534,8 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
require.Len(t, repo.metadataUpdates, 1)
_, present := repo.metadataUpdates[0]["source_updated_at"]
assert.False(t, present)
_, present = repo.metadataUpdates[0]["source_created_at"]
assert.False(t, present)
}
func TestProcessSync_SyncDeletionsDeletesMatchingKnowledge(t *testing.T) {

View File

@ -290,11 +290,13 @@ type DocxFetchInput struct {
// WeKnora external_id: wiki=node.NodeToken, drive=file.Token
DocToken string
// Feishu docx document token
ObjToken string
Title string
URL string
ResourceID string
EditTime time.Time
ObjToken string
Title string
URL string
ResourceID string
EditTime time.Time
// CreateTime is the document creation time in Feishu; zero when unknown.
CreateTime time.Time
BaseMeta map[string]string
MultimodalEnabled bool
}
@ -364,6 +366,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
FileName: SanitizeFileName(in.Title) + ".md",
URL: in.URL,
UpdatedAt: in.EditTime,
CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: in.BaseMeta,
ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync
@ -414,6 +417,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
FileName: SanitizeFileName(a.Name),
URL: in.URL,
UpdatedAt: in.EditTime,
CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: childMeta(),
})
@ -463,6 +467,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
FileName: "image-" + b.Image.Token + ext,
URL: in.URL,
UpdatedAt: in.EditTime,
CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: imgMeta(),
})
@ -496,6 +501,7 @@ func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput)
FileName: fileName,
URL: in.URL,
UpdatedAt: in.EditTime,
CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: in.BaseMeta,
}, nil

View File

@ -363,6 +363,7 @@ func fetchDriveFileContent(
}
editTime := core.ParseFeishuTimestamp(file.ModifiedTime)
createTime := core.ParseFeishuTimestamp(file.CreatedTime)
// Channel marks the knowledge "source" label. Drive uses its own channel
// (feishu_drive / lark_drive) so Drive docs show "飞书云盘" / "Lark 云盘"
// distinct from the wiki connector's "飞书".
@ -387,6 +388,7 @@ func fetchDriveFileContent(
URL: file.URL,
ResourceID: resourceID,
EditTime: editTime,
CreateTime: createTime,
BaseMeta: baseMeta,
MultimodalEnabled: multimodalEnabled,
})
@ -412,6 +414,7 @@ func fetchDriveFileContent(
FileName: fileName,
URL: file.URL,
UpdatedAt: editTime,
CreatedAt: createTime,
SourceResourceID: resourceID,
Metadata: baseMeta,
}}, nil
@ -435,6 +438,7 @@ func fetchDriveFileContent(
FileName: fileName,
URL: file.URL,
UpdatedAt: editTime,
CreatedAt: createTime,
SourceResourceID: resourceID,
Metadata: baseMeta,
}}, nil

View File

@ -243,8 +243,9 @@ func (o wikiOps) Title(n core.WikiNode) string { return n.Title }
func (o wikiOps) ObjType(n core.WikiNode) string { return n.ObjType }
// EditTime is the change-detection timestamp: ObjEditTime (document content)
// with a NodeEditTime fallback for nodes that lack obj_edit_time. This drives
// the cursor comparison, NOT FetchedItem.UpdatedAt (which uses NodeEditTime).
// with a NodeEditTime fallback for nodes that lack obj_edit_time. It drives the
// cursor comparison and, parsed, FetchedItem.UpdatedAt (see contentEditTime),
// so the persisted source_updated_at tracks content edits, not node moves.
func (o wikiOps) EditTime(n core.WikiNode) string {
if n.ObjEditTime != "" {
return n.ObjEditTime
@ -323,7 +324,8 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
return nil, nil
}
editTime := core.ParseFeishuTimestamp(node.NodeEditTime)
editTime := contentEditTime(node)
createTime := contentCreateTime(node)
baseMeta := map[string]string{
"obj_token": node.ObjToken,
"obj_type": node.ObjType,
@ -343,6 +345,7 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
URL: region.WikiURL(node.NodeToken),
ResourceID: resourceID,
EditTime: editTime,
CreateTime: createTime,
BaseMeta: baseMeta,
MultimodalEnabled: multimodalEnabled,
})
@ -351,12 +354,14 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
if err != nil {
return nil, err
}
item.CreatedAt = createTime
return []*types.FetchedItem{item}, nil
case "file":
item, err := fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta, region)
if err != nil {
return nil, err
}
item.CreatedAt = createTime
return []*types.FetchedItem{item}, nil
default:
return nil, nil
@ -422,6 +427,26 @@ func fetchDriveFile(ctx context.Context, client *core.Client, node core.WikiNode
}, nil
}
// contentEditTime is the document's last content edit time (obj_edit_time),
// falling back to the node attribute edit time when Feishu omits it. It is
// what ingestion persists as source_updated_at; NodeEditTime alone would move
// on a rename or a move in the tree without any content change.
func contentEditTime(node core.WikiNode) time.Time {
if t := core.ParseFeishuTimestamp(node.ObjEditTime); !t.IsZero() {
return t
}
return core.ParseFeishuTimestamp(node.NodeEditTime)
}
// contentCreateTime is the document creation time (obj_create_time), falling
// back to the node creation time. Persisted as source_created_at.
func contentCreateTime(node core.WikiNode) time.Time {
if t := core.ParseFeishuTimestamp(node.ObjCreateTime); !t.IsZero() {
return t
}
return core.ParseFeishuTimestamp(node.NodeCreateTime)
}
// --- Helper functions ---
func makeWikiNodeResourceID(spaceID, nodeToken string) string {

View File

@ -579,11 +579,14 @@ func TestConnectorResolveResourceAncestors(t *testing.T) {
func TestFetchAll_DocxNode(t *testing.T) {
nodes := []core.WikiNode{{
NodeToken: "nt1",
ObjToken: "obj-docx-1",
ObjType: "docx",
Title: "My Document",
NodeEditTime: "1711468800",
NodeToken: "nt1",
ObjToken: "obj-docx-1",
ObjType: "docx",
Title: "My Document",
ObjCreateTime: "1700000000",
ObjEditTime: "1711000000",
NodeCreateTime: "1700000001",
NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
@ -617,6 +620,27 @@ func TestFetchAll_DocxNode(t *testing.T) {
if item.Metadata["channel"] != types.ChannelFeishu {
t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])
}
// Source timestamps come from the document (obj_*), not the wiki node
// attributes, so a rename or move does not look like a content edit.
if got := item.UpdatedAt.Unix(); got != 1711000000 {
t.Errorf("UpdatedAt = %d, want obj_edit_time 1711000000", got)
}
if got := item.CreatedAt.Unix(); got != 1700000000 {
t.Errorf("CreatedAt = %d, want obj_create_time 1700000000", got)
}
}
func TestContentTimes_FallBackToNodeTimes(t *testing.T) {
n := core.WikiNode{NodeCreateTime: "1700000001", NodeEditTime: "1711468800"}
if got := contentEditTime(n).Unix(); got != 1711468800 {
t.Errorf("contentEditTime fallback = %d, want node_edit_time", got)
}
if got := contentCreateTime(n).Unix(); got != 1700000001 {
t.Errorf("contentCreateTime fallback = %d, want node_create_time", got)
}
if !contentEditTime(core.WikiNode{}).IsZero() || !contentCreateTime(core.WikiNode{}).IsZero() {
t.Errorf("missing times must stay zero, not epoch")
}
}
func TestFetchAll_SheetNode(t *testing.T) {

View File

@ -404,7 +404,24 @@ func (c *Connector) item(ctx context.Context, p *project, ref, file string) (typ
return types.FetchedItem{}, err
}
id := fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file)
return types.FetchedItem{ExternalID: id, Title: p.PathWithNamespace + "/" + file, FileName: knowledgeRelativePath(p.Name, ref, file), Content: body, ContentType: "text/plain", UpdatedAt: time.Now().UTC(), SourceResourceID: fmt.Sprint(p.ID), Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "source_type": "gitlab", "gitlab_project_id": fmt.Sprint(p.ID), "gitlab_ref": ref, "gitlab_path": file, "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file}}, nil
// UpdatedAt is intentionally left unset: the file's last commit time is not
// fetched here, and a fetch timestamp would be a fabricated source time.
return types.FetchedItem{
ExternalID: id,
Title: p.PathWithNamespace + "/" + file,
FileName: knowledgeRelativePath(p.Name, ref, file),
Content: body,
ContentType: "text/plain",
SourceResourceID: fmt.Sprint(p.ID),
Metadata: map[string]string{
"channel": types.ConnectorTypeGitLab,
"source_type": "gitlab",
"gitlab_project_id": fmt.Sprint(p.ID),
"gitlab_ref": ref,
"gitlab_path": file,
"gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file,
},
}, nil
}
// knowledgeRelativePath maps a repository file to the KB folder convention:

View File

@ -332,6 +332,9 @@ type FetchedItem struct {
// When last modified in external system
UpdatedAt time.Time `json:"updated_at"`
// When created in external system. Zero when the source does not expose it.
CreatedAt time.Time `json:"created_at"`
// Additional metadata to preserve
Metadata map[string]string `json:"metadata"`