fix(datasource): validate cron schedules and persist manual-only mode (#3653)

* fix(datasource): preserve schedule state on invalid updates

* docs(datasource): document PUT /datasource/:id as a full replace

The update now always persists sync_schedule, so a partial body clears
the schedule. Show a full-object example and state that omitted fields
are reset rather than preserved.

---------

Co-authored-by: wizardchen <wizardchen@tencent.com>
main
Perfecto 2026-09-25 22:23:18 +08:00 committed by GitHub
parent a9d71038b6
commit 00e6366fd2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 346 additions and 4 deletions

View File

@ -90,10 +90,13 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource)
return err
}
// GORM Updates(struct) deliberately skips zero values, which would make
// a user-selected sync_deletions=false impossible to persist.
// a user-selected sync_deletions=false or manual-only schedule impossible to persist.
return tx.Model(&types.DataSource{}).
Where("id = ?", ds.ID).
UpdateColumn("sync_deletions", ds.SyncDeletions).Error
UpdateColumns(map[string]interface{}{
"sync_deletions": ds.SyncDeletions,
"sync_schedule": ds.SyncSchedule,
}).Error
})
}

View File

@ -0,0 +1,133 @@
package service
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
func TestDataSourceResumeRejectsInvalidSchedule(t *testing.T) {
f := newSQLiteDataSourceDeleteFixture(t)
f.scheduler.Remove(f.ds.ID)
f.ds.Status = types.DataSourceStatusPaused
f.ds.SyncSchedule = "not-a-cron"
require.NoError(t, f.dsRepo.Update(context.Background(), f.ds))
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
err := svc.ResumeDataSource(context.Background(), f.ds.ID)
saved, readErr := svc.GetDataSource(context.Background(), f.ds.ID)
require.NoError(t, readErr)
t.Logf("resume error=%v; status=%s; entries=%d", err, saved.Status, f.scheduler.EntryCount())
require.ErrorContains(t, err, "invalid cron expression")
require.Equal(t, types.DataSourceStatusPaused, saved.Status)
require.Zero(t, f.scheduler.EntryCount())
}
func TestDataSourceUpdateRejectsInvalidSchedule(t *testing.T) {
f := newSQLiteDataSourceDeleteFixture(t)
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
updated := *f.ds
updated.SyncSchedule = "not-a-cron"
_, err := svc.UpdateDataSource(context.Background(), &updated)
require.ErrorContains(t, err, "invalid cron expression")
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
require.NoError(t, err)
require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)
require.Equal(t, 1, f.scheduler.EntryCount())
}
func TestDataSourceCreateRejectsInvalidSchedule(t *testing.T) {
f := newSQLiteDataSourceDeleteFixture(t)
registry := datasource.NewConnectorRegistry()
require.NoError(t, registry.Register(deletedItemConnector{}))
svc := &DataSourceService{
dsRepo: f.dsRepo, scheduler: f.scheduler, connectorRegistry: registry,
kbService: &processSyncKBService{kb: &types.KnowledgeBase{ID: f.ds.KnowledgeBaseID, TenantID: f.ds.TenantID}},
}
ds := *f.ds
ds.ID = "invalid-schedule"
ds.Type = deletedItemConnectorType
ds.SyncSchedule = "not-a-cron"
ds.Config = []byte(`{"type":"test"}`)
_, err := svc.CreateDataSource(context.Background(), &ds)
require.ErrorContains(t, err, "invalid cron expression")
_, err = svc.GetDataSource(context.Background(), ds.ID)
require.Error(t, err)
require.Equal(t, 1, f.scheduler.EntryCount())
}
func TestDataSourceScheduleLifecycle(t *testing.T) {
for _, schedule := range []string{"", "0 0 */6 * * *", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {
t.Run(schedule, func(t *testing.T) {
f := newSQLiteDataSourceDeleteFixture(t)
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
updated := *f.ds
updated.SyncSchedule = schedule
_, err := svc.UpdateDataSource(context.Background(), &updated)
require.NoError(t, err)
wantEntries := 1
if schedule == "" {
wantEntries = 0
}
require.Equal(t, wantEntries, f.scheduler.EntryCount())
require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))
require.Zero(t, f.scheduler.EntryCount())
// Repeated resume must replace, rather than duplicate, the job.
for range 2 {
require.NoError(t, svc.ResumeDataSource(context.Background(), f.ds.ID))
require.Equal(t, wantEntries, f.scheduler.EntryCount())
}
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
require.NoError(t, err)
require.Equal(t, types.DataSourceStatusActive, saved.Status)
require.Equal(t, schedule, saved.SyncSchedule)
})
}
}
func TestDataSourceScheduleWriteFailurePreservesState(t *testing.T) {
for _, operation := range []string{"update", "resume"} {
t.Run(operation, func(t *testing.T) {
f := newSQLiteDataSourceDeleteFixture(t)
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
if operation == "resume" {
require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))
}
before, err := svc.GetDataSource(context.Background(), f.ds.ID)
require.NoError(t, err)
entries := f.scheduler.EntryCount()
require.NoError(t, f.db.Exec(`CREATE TRIGGER fail_schedule_write BEFORE UPDATE ON data_sources
BEGIN SELECT RAISE(FAIL, 'forced schedule write failure'); END;`).Error)
if operation == "resume" {
err = svc.ResumeDataSource(context.Background(), f.ds.ID)
} else {
updated := *before
updated.SyncSchedule = "0 30 * * * *"
_, err = svc.UpdateDataSource(context.Background(), &updated)
}
require.ErrorContains(t, err, "forced schedule write failure")
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
require.NoError(t, err)
require.Equal(t, before.Status, saved.Status)
require.Equal(t, before.SyncSchedule, saved.SyncSchedule)
require.Equal(t, entries, f.scheduler.EntryCount())
})
}
}
func TestDataSourceScheduleCanceledUpdatePreservesState(t *testing.T) {
f := newSQLiteDataSourceDeleteFixture(t)
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
updated := *f.ds
updated.SyncSchedule = "0 30 * * * *"
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := svc.UpdateDataSource(ctx, &updated)
require.ErrorIs(t, err, context.Canceled)
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
require.NoError(t, err)
require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)
require.Equal(t, 1, f.scheduler.EntryCount())
}

View File

@ -69,6 +69,9 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Data
if ds == nil {
return nil, datasource.ErrDataSourceInvalid
}
if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
return nil, err
}
// Validate knowledge base exists
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID)
@ -169,6 +172,9 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
if ds.TenantID != existing.TenantID {
return nil, datasource.ErrDataSourceInvalid
}
if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
return nil, err
}
// Credentials NEVER flow through this endpoint — they live behind the
// /credentials subresource. Force-preserve the stored credentials map
@ -549,6 +555,9 @@ func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) err
if err != nil {
return err
}
if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
return err
}
ds.Status = types.DataSourceStatusActive
if err := s.dsRepo.Update(ctx, ds); err != nil {

View File

@ -15,6 +15,22 @@ import (
"github.com/robfig/cron/v3"
)
var syncScheduleParser = cron.NewParser(
cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
)
// ValidateSyncSchedule accepts the same syntax as the scheduler. An empty
// schedule disables automatic sync and remains valid for manual-only sources.
func ValidateSyncSchedule(schedule string) error {
if schedule == "" {
return nil
}
if _, err := syncScheduleParser.Parse(schedule); err != nil {
return fmt.Errorf("invalid cron expression %q: %w", schedule, err)
}
return nil
}
// Scheduler manages cron-based periodic sync for data sources.
//
// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires
@ -41,7 +57,7 @@ func NewScheduler(
taskEnqueuer interfaces.TaskEnqueuer,
) *Scheduler {
return &Scheduler{
cron: cron.New(cron.WithSeconds(), cron.WithChain(
cron: cron.New(cron.WithParser(syncScheduleParser), cron.WithChain(
cron.Recover(cron.DefaultLogger),
)),
dsRepo: dsRepo,
@ -82,6 +98,13 @@ func (s *Scheduler) Stop() {
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error {
// Reject an invalid replacement before removing the working entry. Inactive
// sources must still be removable even if their stored schedule is invalid.
if ds.Status == types.DataSourceStatusActive {
if err := ValidateSyncSchedule(ds.SyncSchedule); err != nil {
return err
}
}
s.mu.Lock()
defer s.mu.Unlock()

View File

@ -345,6 +345,92 @@ func TestScheduler_InvalidCron(t *testing.T) {
}
}
func TestScheduler_InvalidReplacementPreservesSchedule(t *testing.T) {
repo := newFakeDataSourceRepo()
enqueuer := &fakeTaskEnqueuer{}
s := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
ds := &types.DataSource{ID: "replace", Status: types.DataSourceStatusActive, SyncSchedule: "* * * * * *"}
if err := repo.Create(context.Background(), ds); err != nil {
t.Fatal(err)
}
if err := s.AddOrUpdate(ds); err != nil {
t.Fatal(err)
}
invalid := *ds
invalid.SyncSchedule = "not-a-cron"
if err := s.AddOrUpdate(&invalid); err == nil {
t.Fatal("expected invalid schedule to be rejected")
}
if got := s.EntryCount(); got != 1 {
t.Fatalf("invalid replacement removed existing schedule: entries=%d", got)
}
s.cron.Start()
defer s.Stop()
deadline := time.Now().Add(3 * time.Second)
for enqueuer.count.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if enqueuer.count.Load() == 0 {
t.Fatal("preserved schedule did not enqueue a task")
}
}
func TestValidateSyncSchedule(t *testing.T) {
for _, schedule := range []string{"", "0 0 */6 * * *", "@hourly", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {
t.Run(schedule, func(t *testing.T) {
if err := ValidateSyncSchedule(schedule); err != nil {
t.Fatal(err)
}
})
}
for _, schedule := range []string{"not-a-cron", "0 0 * * *", "60 * * * * *", "CRON_TZ=Invalid/Zone 0 0 * * * *"} {
t.Run(schedule, func(t *testing.T) {
if err := ValidateSyncSchedule(schedule); err == nil {
t.Fatal("expected invalid schedule to be rejected")
}
})
}
}
func TestScheduler_ConcurrentReplacements(t *testing.T) {
s := NewScheduler(nil, nil, nil)
var wg sync.WaitGroup
for i := range 20 {
wg.Add(1)
go func() {
defer wg.Done()
schedule := "0 0 * * * *"
if i%2 == 0 {
schedule = "invalid"
}
ds := &types.DataSource{ID: "concurrent", Status: types.DataSourceStatusActive, SyncSchedule: schedule}
if err := s.AddOrUpdate(ds); (err != nil) != (i%2 == 0) {
t.Errorf("unexpected result for schedule %q: %v", schedule, err)
}
}()
}
wg.Wait()
if got := s.EntryCount(); got != 1 {
t.Fatalf("concurrent replacements left %d entries, want 1", got)
}
}
func TestScheduler_PauseRemovesEntryWithInvalidStoredSchedule(t *testing.T) {
s := NewScheduler(nil, nil, nil)
ds := &types.DataSource{ID: "pause", Status: types.DataSourceStatusActive, SyncSchedule: "0 0 * * * *"}
if err := s.AddOrUpdate(ds); err != nil {
t.Fatal(err)
}
ds.Status = types.DataSourceStatusPaused
ds.SyncSchedule = "invalid"
if err := s.AddOrUpdate(ds); err != nil {
t.Fatal(err)
}
if got := s.EntryCount(); got != 0 {
t.Fatalf("pause left %d entries", got)
}
}
func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) {
repo := newFakeDataSourceRepo()
// Create a data source that is paused

View File

@ -0,0 +1,80 @@
package handler
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func TestDataSourceScheduleAPIRejectsInvalidCron(t *testing.T) {
for _, operation := range []string{"create", "update", "resume"} {
t.Run(operation, func(t *testing.T) {
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "schedule.db")), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.DataSource{}))
repo := repository.NewDataSourceRepository(db)
scheduler := datasource.NewScheduler(repo, nil, nil)
kbSvc := &stubKBServiceForDS{getByID: func(context.Context, string) (*types.KnowledgeBase, error) {
return &types.KnowledgeBase{ID: "kb-schedule", TenantID: 1}, nil
}}
svc := service.NewDataSourceService(repo, nil, nil, kbSvc, nil, nil, scheduler, nil, nil, nil)
ds := &types.DataSource{
ID: "ds-schedule", KnowledgeBaseID: "kb-schedule", TenantID: 1,
Type: types.ConnectorTypeRSS, Status: types.DataSourceStatusActive,
SyncSchedule: "0 0 * * * *",
}
if operation == "resume" {
ds.Status = types.DataSourceStatusPaused
ds.SyncSchedule = "invalid"
}
if operation != "create" {
require.NoError(t, repo.Create(context.Background(), ds))
require.NoError(t, scheduler.AddOrUpdate(ds))
}
entries := scheduler.EntryCount()
h := NewDataSourceHandler(svc, kbSvc)
router := newDataSourceTestRouter(h)
router.POST("/datasource", h.CreateDataSource)
router.PUT("/datasource/:id", h.UpdateDataSource)
router.POST("/datasource/:id/resume", h.ResumeDataSource)
body := *ds
body.SyncSchedule = "invalid"
payload, err := json.Marshal(body)
require.NoError(t, err)
method, url := http.MethodPost, "/datasource"
switch operation {
case "update":
method, url = http.MethodPut, "/datasource/"+ds.ID
case "resume":
url = "/datasource/" + ds.ID + "/resume"
}
req := withDSCtx(httptest.NewRequest(method, url, bytes.NewReader(payload)), 1)
req.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
router.ServeHTTP(response, req)
require.Equal(t, http.StatusBadRequest, response.Code, response.Body.String())
require.Contains(t, response.Body.String(), "invalid cron expression")
require.Equal(t, entries, scheduler.EntryCount())
saved, err := svc.GetDataSource(context.Background(), ds.ID)
if operation == "create" {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, ds.Status, saved.Status)
require.Equal(t, ds.SyncSchedule, saved.SyncSchedule)
}
})
}
}

View File

@ -20,6 +20,8 @@
凭据更新需要提交完整配置,系统会在线验证。暂停数据源可停止后续计划同步;恢复后重新注册调度。
`sync_schedule` 使用包含秒的六段 cron,例如 `0 0 */6 * * *` 表示每六小时触发;空字符串表示仅手动同步。创建、编辑及恢复数据源时会校验表达式,无效值返回 HTTP 400,不改变已保存配置或已有定时任务。历史数据源若保存了无效表达式,需先修改为合法表达式或清空,再恢复同步。
## 选择连接器
| 连接器 | 类型标识 | 同步对象 |

View File

@ -374,11 +374,17 @@ curl $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN"
用途:更新(`id/tenant_id/knowledge_base_id` 锁定为原值)。权限:Admin+。请求体同创建。
PUT 整体替换数据源配置,不支持部分更新:请先 GET 详情,修改后回传完整对象(`name`、`type`、`config`、`sync_schedule`、`sync_mode`、`conflict_strategy`、`sync_deletions`、`sync_log_retention_days`、`status` 等)。`sync_schedule` 和 `sync_deletions` 总是按请求体写入:省略 `sync_schedule` 会清空为空字符串(仅手动同步,已有定时任务随之移除),省略 `sync_deletions` 即为 false;其他字段省略时是否保留原值不作保证。凭证不经本接口修改,请求体中的 `config.credentials` 会被忽略,保留已存凭证。
响应:200 `DataSourceResponse`
```bash
curl -X PUT $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1","config":{}}'
-H 'Content-Type: application/json' \
-d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1",
"config":{"type":"notion","resource_ids":["page-1"]},
"sync_schedule":"0 0 */6 * * *","sync_mode":"incremental","conflict_strategy":"overwrite",
"sync_deletions":true,"sync_log_retention_days":30,"status":"active"}'
```
### DELETE /api/v1/datasource/:id