feat(datasource): persist source created time; Feishu uses content edit time for source_updated_at (#3083)
* feat(datasource): persist source document creation time and use content edit time for Feishu
Follow-up to 160c806b (Refs #3016), which started persisting the source
last-modified time as metadata.source_updated_at. Two gaps remained:
1. Creation time was never carried. FetchedItem gains CreatedAt and
ingestItem persists it as metadata.source_created_at (RFC3339, UTC)
with the same absent-means-unknown rule as source_updated_at.
2. The Feishu wiki connector filled FetchedItem.UpdatedAt from
NodeEditTime, which moves when a node is renamed or dragged in the
tree without any content change. The connector's own cursor already
prefers ObjEditTime for exactly this reason; use the same rule for the
persisted value (contentEditTime / contentCreateTime, node_* as
fallback), so source_updated_at reflects document edits.
Feishu Drive populates CreatedAt from created_time. GitLab stops
fabricating UpdatedAt = time.Now(): a fetch timestamp is not a document
timestamp, and an absent value is the documented "unknown" (the commit
time is not fetched today; leaving it unset is honest until it is).
Tests: ingestItem covers both keys present/absent; the wiki fetch test
asserts obj_* times win and node_* are the fallback.
* style(gitlab): break up the FetchedItem literal (lll)
---------
Co-authored-by: xiaoyaner0201 <>
main
parent
6a55860431
commit
061df8efdc
|
|
@ -1257,6 +1257,9 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
|
|||
if !item.UpdatedAt.IsZero() {
|
||||
metadata["source_updated_at"] = item.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if !item.CreatedAt.IsZero() {
|
||||
metadata["source_created_at"] = item.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
for k, v := range item.Metadata {
|
||||
metadata[k] = v
|
||||
}
|
||||
|
|
|
|||
|
|
@ -511,14 +511,17 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
|
|||
repo := &deletionLookupKnowledgeRepo{}
|
||||
ks := &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-1"}}
|
||||
svc := &DataSourceService{knowledgeService: ks}
|
||||
created := time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
_, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{
|
||||
ExternalID: "url:1",
|
||||
URL: "https://example.com/doc",
|
||||
UpdatedAt: edited,
|
||||
CreatedAt: created,
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, repo.metadataUpdates, 1)
|
||||
assert.Equal(t, "2023-04-04T22:07:08Z", repo.metadataUpdates[0]["source_updated_at"])
|
||||
assert.Equal(t, "2021-01-02T03:04:05Z", repo.metadataUpdates[0]["source_created_at"])
|
||||
|
||||
repo = &deletionLookupKnowledgeRepo{}
|
||||
ks = &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-2"}}
|
||||
|
|
@ -531,6 +534,8 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
|
|||
require.Len(t, repo.metadataUpdates, 1)
|
||||
_, present := repo.metadataUpdates[0]["source_updated_at"]
|
||||
assert.False(t, present)
|
||||
_, present = repo.metadataUpdates[0]["source_created_at"]
|
||||
assert.False(t, present)
|
||||
}
|
||||
|
||||
func TestProcessSync_SyncDeletionsDeletesMatchingKnowledge(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -290,11 +290,13 @@ type DocxFetchInput struct {
|
|||
// WeKnora external_id: wiki=node.NodeToken, drive=file.Token
|
||||
DocToken string
|
||||
// Feishu docx document token
|
||||
ObjToken string
|
||||
Title string
|
||||
URL string
|
||||
ResourceID string
|
||||
EditTime time.Time
|
||||
ObjToken string
|
||||
Title string
|
||||
URL string
|
||||
ResourceID string
|
||||
EditTime time.Time
|
||||
// CreateTime is the document creation time in Feishu; zero when unknown.
|
||||
CreateTime time.Time
|
||||
BaseMeta map[string]string
|
||||
MultimodalEnabled bool
|
||||
}
|
||||
|
|
@ -364,6 +366,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
|
|||
FileName: SanitizeFileName(in.Title) + ".md",
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
CreatedAt: in.CreateTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: in.BaseMeta,
|
||||
ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync
|
||||
|
|
@ -414,6 +417,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
|
|||
FileName: SanitizeFileName(a.Name),
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
CreatedAt: in.CreateTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: childMeta(),
|
||||
})
|
||||
|
|
@ -463,6 +467,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
|
|||
FileName: "image-" + b.Image.Token + ext,
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
CreatedAt: in.CreateTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: imgMeta(),
|
||||
})
|
||||
|
|
@ -496,6 +501,7 @@ func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput)
|
|||
FileName: fileName,
|
||||
URL: in.URL,
|
||||
UpdatedAt: in.EditTime,
|
||||
CreatedAt: in.CreateTime,
|
||||
SourceResourceID: in.ResourceID,
|
||||
Metadata: in.BaseMeta,
|
||||
}, nil
|
||||
|
|
|
|||
|
|
@ -363,6 +363,7 @@ func fetchDriveFileContent(
|
|||
}
|
||||
|
||||
editTime := core.ParseFeishuTimestamp(file.ModifiedTime)
|
||||
createTime := core.ParseFeishuTimestamp(file.CreatedTime)
|
||||
// Channel marks the knowledge "source" label. Drive uses its own channel
|
||||
// (feishu_drive / lark_drive) so Drive docs show "飞书云盘" / "Lark 云盘"
|
||||
// distinct from the wiki connector's "飞书".
|
||||
|
|
@ -387,6 +388,7 @@ func fetchDriveFileContent(
|
|||
URL: file.URL,
|
||||
ResourceID: resourceID,
|
||||
EditTime: editTime,
|
||||
CreateTime: createTime,
|
||||
BaseMeta: baseMeta,
|
||||
MultimodalEnabled: multimodalEnabled,
|
||||
})
|
||||
|
|
@ -412,6 +414,7 @@ func fetchDriveFileContent(
|
|||
FileName: fileName,
|
||||
URL: file.URL,
|
||||
UpdatedAt: editTime,
|
||||
CreatedAt: createTime,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: baseMeta,
|
||||
}}, nil
|
||||
|
|
@ -435,6 +438,7 @@ func fetchDriveFileContent(
|
|||
FileName: fileName,
|
||||
URL: file.URL,
|
||||
UpdatedAt: editTime,
|
||||
CreatedAt: createTime,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: baseMeta,
|
||||
}}, nil
|
||||
|
|
|
|||
|
|
@ -243,8 +243,9 @@ func (o wikiOps) Title(n core.WikiNode) string { return n.Title }
|
|||
func (o wikiOps) ObjType(n core.WikiNode) string { return n.ObjType }
|
||||
|
||||
// EditTime is the change-detection timestamp: ObjEditTime (document content)
|
||||
// with a NodeEditTime fallback for nodes that lack obj_edit_time. This drives
|
||||
// the cursor comparison, NOT FetchedItem.UpdatedAt (which uses NodeEditTime).
|
||||
// with a NodeEditTime fallback for nodes that lack obj_edit_time. It drives the
|
||||
// cursor comparison and, parsed, FetchedItem.UpdatedAt (see contentEditTime),
|
||||
// so the persisted source_updated_at tracks content edits, not node moves.
|
||||
func (o wikiOps) EditTime(n core.WikiNode) string {
|
||||
if n.ObjEditTime != "" {
|
||||
return n.ObjEditTime
|
||||
|
|
@ -323,7 +324,8 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
|
|||
return nil, nil
|
||||
}
|
||||
|
||||
editTime := core.ParseFeishuTimestamp(node.NodeEditTime)
|
||||
editTime := contentEditTime(node)
|
||||
createTime := contentCreateTime(node)
|
||||
baseMeta := map[string]string{
|
||||
"obj_token": node.ObjToken,
|
||||
"obj_type": node.ObjType,
|
||||
|
|
@ -343,6 +345,7 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
|
|||
URL: region.WikiURL(node.NodeToken),
|
||||
ResourceID: resourceID,
|
||||
EditTime: editTime,
|
||||
CreateTime: createTime,
|
||||
BaseMeta: baseMeta,
|
||||
MultimodalEnabled: multimodalEnabled,
|
||||
})
|
||||
|
|
@ -351,12 +354,14 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.CreatedAt = createTime
|
||||
return []*types.FetchedItem{item}, nil
|
||||
case "file":
|
||||
item, err := fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta, region)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.CreatedAt = createTime
|
||||
return []*types.FetchedItem{item}, nil
|
||||
default:
|
||||
return nil, nil
|
||||
|
|
@ -422,6 +427,26 @@ func fetchDriveFile(ctx context.Context, client *core.Client, node core.WikiNode
|
|||
}, nil
|
||||
}
|
||||
|
||||
// contentEditTime is the document's last content edit time (obj_edit_time),
|
||||
// falling back to the node attribute edit time when Feishu omits it. It is
|
||||
// what ingestion persists as source_updated_at; NodeEditTime alone would move
|
||||
// on a rename or a move in the tree without any content change.
|
||||
func contentEditTime(node core.WikiNode) time.Time {
|
||||
if t := core.ParseFeishuTimestamp(node.ObjEditTime); !t.IsZero() {
|
||||
return t
|
||||
}
|
||||
return core.ParseFeishuTimestamp(node.NodeEditTime)
|
||||
}
|
||||
|
||||
// contentCreateTime is the document creation time (obj_create_time), falling
|
||||
// back to the node creation time. Persisted as source_created_at.
|
||||
func contentCreateTime(node core.WikiNode) time.Time {
|
||||
if t := core.ParseFeishuTimestamp(node.ObjCreateTime); !t.IsZero() {
|
||||
return t
|
||||
}
|
||||
return core.ParseFeishuTimestamp(node.NodeCreateTime)
|
||||
}
|
||||
|
||||
// --- Helper functions ---
|
||||
|
||||
func makeWikiNodeResourceID(spaceID, nodeToken string) string {
|
||||
|
|
|
|||
|
|
@ -579,11 +579,14 @@ func TestConnectorResolveResourceAncestors(t *testing.T) {
|
|||
|
||||
func TestFetchAll_DocxNode(t *testing.T) {
|
||||
nodes := []core.WikiNode{{
|
||||
NodeToken: "nt1",
|
||||
ObjToken: "obj-docx-1",
|
||||
ObjType: "docx",
|
||||
Title: "My Document",
|
||||
NodeEditTime: "1711468800",
|
||||
NodeToken: "nt1",
|
||||
ObjToken: "obj-docx-1",
|
||||
ObjType: "docx",
|
||||
Title: "My Document",
|
||||
ObjCreateTime: "1700000000",
|
||||
ObjEditTime: "1711000000",
|
||||
NodeCreateTime: "1700000001",
|
||||
NodeEditTime: "1711468800",
|
||||
}}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
|
@ -617,6 +620,27 @@ func TestFetchAll_DocxNode(t *testing.T) {
|
|||
if item.Metadata["channel"] != types.ChannelFeishu {
|
||||
t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])
|
||||
}
|
||||
// Source timestamps come from the document (obj_*), not the wiki node
|
||||
// attributes, so a rename or move does not look like a content edit.
|
||||
if got := item.UpdatedAt.Unix(); got != 1711000000 {
|
||||
t.Errorf("UpdatedAt = %d, want obj_edit_time 1711000000", got)
|
||||
}
|
||||
if got := item.CreatedAt.Unix(); got != 1700000000 {
|
||||
t.Errorf("CreatedAt = %d, want obj_create_time 1700000000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentTimes_FallBackToNodeTimes(t *testing.T) {
|
||||
n := core.WikiNode{NodeCreateTime: "1700000001", NodeEditTime: "1711468800"}
|
||||
if got := contentEditTime(n).Unix(); got != 1711468800 {
|
||||
t.Errorf("contentEditTime fallback = %d, want node_edit_time", got)
|
||||
}
|
||||
if got := contentCreateTime(n).Unix(); got != 1700000001 {
|
||||
t.Errorf("contentCreateTime fallback = %d, want node_create_time", got)
|
||||
}
|
||||
if !contentEditTime(core.WikiNode{}).IsZero() || !contentCreateTime(core.WikiNode{}).IsZero() {
|
||||
t.Errorf("missing times must stay zero, not epoch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_SheetNode(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -404,7 +404,24 @@ func (c *Connector) item(ctx context.Context, p *project, ref, file string) (typ
|
|||
return types.FetchedItem{}, err
|
||||
}
|
||||
id := fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file)
|
||||
return types.FetchedItem{ExternalID: id, Title: p.PathWithNamespace + "/" + file, FileName: knowledgeRelativePath(p.Name, ref, file), Content: body, ContentType: "text/plain", UpdatedAt: time.Now().UTC(), SourceResourceID: fmt.Sprint(p.ID), Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "source_type": "gitlab", "gitlab_project_id": fmt.Sprint(p.ID), "gitlab_ref": ref, "gitlab_path": file, "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file}}, nil
|
||||
// UpdatedAt is intentionally left unset: the file's last commit time is not
|
||||
// fetched here, and a fetch timestamp would be a fabricated source time.
|
||||
return types.FetchedItem{
|
||||
ExternalID: id,
|
||||
Title: p.PathWithNamespace + "/" + file,
|
||||
FileName: knowledgeRelativePath(p.Name, ref, file),
|
||||
Content: body,
|
||||
ContentType: "text/plain",
|
||||
SourceResourceID: fmt.Sprint(p.ID),
|
||||
Metadata: map[string]string{
|
||||
"channel": types.ConnectorTypeGitLab,
|
||||
"source_type": "gitlab",
|
||||
"gitlab_project_id": fmt.Sprint(p.ID),
|
||||
"gitlab_ref": ref,
|
||||
"gitlab_path": file,
|
||||
"gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// knowledgeRelativePath maps a repository file to the KB folder convention:
|
||||
|
|
|
|||
|
|
@ -332,6 +332,9 @@ type FetchedItem struct {
|
|||
// When last modified in external system
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
|
||||
// When created in external system. Zero when the source does not expose it.
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
||||
// Additional metadata to preserve
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue