diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go index c3b2f58e..8f2cabbe 100644 --- a/internal/application/repository/datasource_repo.go +++ b/internal/application/repository/datasource_repo.go @@ -90,10 +90,13 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource) return err } // GORM Updates(struct) deliberately skips zero values, which would make - // a user-selected sync_deletions=false impossible to persist. + // a user-selected sync_deletions=false or manual-only schedule impossible to persist. return tx.Model(&types.DataSource{}). Where("id = ?", ds.ID). - UpdateColumn("sync_deletions", ds.SyncDeletions).Error + UpdateColumns(map[string]interface{}{ + "sync_deletions": ds.SyncDeletions, + "sync_schedule": ds.SyncSchedule, + }).Error }) } diff --git a/internal/application/service/datasource_schedule_test.go b/internal/application/service/datasource_schedule_test.go new file mode 100644 index 00000000..0320bd3f --- /dev/null +++ b/internal/application/service/datasource_schedule_test.go @@ -0,0 +1,133 @@ +package service + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestDataSourceResumeRejectsInvalidSchedule(t *testing.T) { + f := newSQLiteDataSourceDeleteFixture(t) + f.scheduler.Remove(f.ds.ID) + f.ds.Status = types.DataSourceStatusPaused + f.ds.SyncSchedule = "not-a-cron" + require.NoError(t, f.dsRepo.Update(context.Background(), f.ds)) + svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler} + err := svc.ResumeDataSource(context.Background(), f.ds.ID) + saved, readErr := svc.GetDataSource(context.Background(), f.ds.ID) + require.NoError(t, readErr) + t.Logf("resume error=%v; status=%s; entries=%d", err, saved.Status, f.scheduler.EntryCount()) + require.ErrorContains(t, err, "invalid cron expression") + require.Equal(t, types.DataSourceStatusPaused, saved.Status) + require.Zero(t, f.scheduler.EntryCount()) +} + +func TestDataSourceUpdateRejectsInvalidSchedule(t *testing.T) { + f := newSQLiteDataSourceDeleteFixture(t) + svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler} + updated := *f.ds + updated.SyncSchedule = "not-a-cron" + _, err := svc.UpdateDataSource(context.Background(), &updated) + require.ErrorContains(t, err, "invalid cron expression") + saved, err := svc.GetDataSource(context.Background(), f.ds.ID) + require.NoError(t, err) + require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule) + require.Equal(t, 1, f.scheduler.EntryCount()) +} + +func TestDataSourceCreateRejectsInvalidSchedule(t *testing.T) { + f := newSQLiteDataSourceDeleteFixture(t) + registry := datasource.NewConnectorRegistry() + require.NoError(t, registry.Register(deletedItemConnector{})) + svc := &DataSourceService{ + dsRepo: f.dsRepo, scheduler: f.scheduler, connectorRegistry: registry, + kbService: &processSyncKBService{kb: &types.KnowledgeBase{ID: f.ds.KnowledgeBaseID, TenantID: f.ds.TenantID}}, + } + ds := *f.ds + ds.ID = "invalid-schedule" + ds.Type = deletedItemConnectorType + ds.SyncSchedule = "not-a-cron" + ds.Config = []byte(`{"type":"test"}`) + _, err := svc.CreateDataSource(context.Background(), &ds) + require.ErrorContains(t, err, "invalid cron expression") + _, err = svc.GetDataSource(context.Background(), ds.ID) + require.Error(t, err) + require.Equal(t, 1, f.scheduler.EntryCount()) +} + +func TestDataSourceScheduleLifecycle(t *testing.T) { + for _, schedule := range []string{"", "0 0 */6 * * *", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} { + t.Run(schedule, func(t *testing.T) { + f := newSQLiteDataSourceDeleteFixture(t) + svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler} + updated := *f.ds + updated.SyncSchedule = schedule + _, err := svc.UpdateDataSource(context.Background(), &updated) + require.NoError(t, err) + wantEntries := 1 + if schedule == "" { + wantEntries = 0 + } + require.Equal(t, wantEntries, f.scheduler.EntryCount()) + require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID)) + require.Zero(t, f.scheduler.EntryCount()) + // Repeated resume must replace, rather than duplicate, the job. + for range 2 { + require.NoError(t, svc.ResumeDataSource(context.Background(), f.ds.ID)) + require.Equal(t, wantEntries, f.scheduler.EntryCount()) + } + saved, err := svc.GetDataSource(context.Background(), f.ds.ID) + require.NoError(t, err) + require.Equal(t, types.DataSourceStatusActive, saved.Status) + require.Equal(t, schedule, saved.SyncSchedule) + }) + } +} + +func TestDataSourceScheduleWriteFailurePreservesState(t *testing.T) { + for _, operation := range []string{"update", "resume"} { + t.Run(operation, func(t *testing.T) { + f := newSQLiteDataSourceDeleteFixture(t) + svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler} + if operation == "resume" { + require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID)) + } + before, err := svc.GetDataSource(context.Background(), f.ds.ID) + require.NoError(t, err) + entries := f.scheduler.EntryCount() + require.NoError(t, f.db.Exec(`CREATE TRIGGER fail_schedule_write BEFORE UPDATE ON data_sources + BEGIN SELECT RAISE(FAIL, 'forced schedule write failure'); END;`).Error) + if operation == "resume" { + err = svc.ResumeDataSource(context.Background(), f.ds.ID) + } else { + updated := *before + updated.SyncSchedule = "0 30 * * * *" + _, err = svc.UpdateDataSource(context.Background(), &updated) + } + require.ErrorContains(t, err, "forced schedule write failure") + saved, err := svc.GetDataSource(context.Background(), f.ds.ID) + require.NoError(t, err) + require.Equal(t, before.Status, saved.Status) + require.Equal(t, before.SyncSchedule, saved.SyncSchedule) + require.Equal(t, entries, f.scheduler.EntryCount()) + }) + } +} + +func TestDataSourceScheduleCanceledUpdatePreservesState(t *testing.T) { + f := newSQLiteDataSourceDeleteFixture(t) + svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler} + updated := *f.ds + updated.SyncSchedule = "0 30 * * * *" + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := svc.UpdateDataSource(ctx, &updated) + require.ErrorIs(t, err, context.Canceled) + saved, err := svc.GetDataSource(context.Background(), f.ds.ID) + require.NoError(t, err) + require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule) + require.Equal(t, 1, f.scheduler.EntryCount()) +} diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 5886da8f..fe106dfe 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -69,6 +69,9 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Data if ds == nil { return nil, datasource.ErrDataSourceInvalid } + if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil { + return nil, err + } // Validate knowledge base exists kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID) @@ -169,6 +172,9 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data if ds.TenantID != existing.TenantID { return nil, datasource.ErrDataSourceInvalid } + if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil { + return nil, err + } // Credentials NEVER flow through this endpoint — they live behind the // /credentials subresource. Force-preserve the stored credentials map @@ -549,6 +555,9 @@ func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) err if err != nil { return err } + if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil { + return err + } ds.Status = types.DataSourceStatusActive if err := s.dsRepo.Update(ctx, ds); err != nil { diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.go index 196327fe..0419d1ca 100644 --- a/internal/datasource/scheduler.go +++ b/internal/datasource/scheduler.go @@ -15,6 +15,22 @@ import ( "github.com/robfig/cron/v3" ) +var syncScheduleParser = cron.NewParser( + cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor, +) + +// ValidateSyncSchedule accepts the same syntax as the scheduler. An empty +// schedule disables automatic sync and remains valid for manual-only sources. +func ValidateSyncSchedule(schedule string) error { + if schedule == "" { + return nil + } + if _, err := syncScheduleParser.Parse(schedule); err != nil { + return fmt.Errorf("invalid cron expression %q: %w", schedule, err) + } + return nil +} + // Scheduler manages cron-based periodic sync for data sources. // // robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires @@ -41,7 +57,7 @@ func NewScheduler( taskEnqueuer interfaces.TaskEnqueuer, ) *Scheduler { return &Scheduler{ - cron: cron.New(cron.WithSeconds(), cron.WithChain( + cron: cron.New(cron.WithParser(syncScheduleParser), cron.WithChain( cron.Recover(cron.DefaultLogger), )), dsRepo: dsRepo, @@ -82,6 +98,13 @@ func (s *Scheduler) Stop() { // AddOrUpdate registers (or re-registers) a cron entry for the given data source. func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error { + // Reject an invalid replacement before removing the working entry. Inactive + // sources must still be removable even if their stored schedule is invalid. + if ds.Status == types.DataSourceStatusActive { + if err := ValidateSyncSchedule(ds.SyncSchedule); err != nil { + return err + } + } s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/datasource/scheduler_test.go b/internal/datasource/scheduler_test.go index dbd35b46..eafa95c8 100644 --- a/internal/datasource/scheduler_test.go +++ b/internal/datasource/scheduler_test.go @@ -345,6 +345,92 @@ func TestScheduler_InvalidCron(t *testing.T) { } } +func TestScheduler_InvalidReplacementPreservesSchedule(t *testing.T) { + repo := newFakeDataSourceRepo() + enqueuer := &fakeTaskEnqueuer{} + s := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer) + ds := &types.DataSource{ID: "replace", Status: types.DataSourceStatusActive, SyncSchedule: "* * * * * *"} + if err := repo.Create(context.Background(), ds); err != nil { + t.Fatal(err) + } + if err := s.AddOrUpdate(ds); err != nil { + t.Fatal(err) + } + invalid := *ds + invalid.SyncSchedule = "not-a-cron" + if err := s.AddOrUpdate(&invalid); err == nil { + t.Fatal("expected invalid schedule to be rejected") + } + if got := s.EntryCount(); got != 1 { + t.Fatalf("invalid replacement removed existing schedule: entries=%d", got) + } + s.cron.Start() + defer s.Stop() + deadline := time.Now().Add(3 * time.Second) + for enqueuer.count.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if enqueuer.count.Load() == 0 { + t.Fatal("preserved schedule did not enqueue a task") + } +} + +func TestValidateSyncSchedule(t *testing.T) { + for _, schedule := range []string{"", "0 0 */6 * * *", "@hourly", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} { + t.Run(schedule, func(t *testing.T) { + if err := ValidateSyncSchedule(schedule); err != nil { + t.Fatal(err) + } + }) + } + for _, schedule := range []string{"not-a-cron", "0 0 * * *", "60 * * * * *", "CRON_TZ=Invalid/Zone 0 0 * * * *"} { + t.Run(schedule, func(t *testing.T) { + if err := ValidateSyncSchedule(schedule); err == nil { + t.Fatal("expected invalid schedule to be rejected") + } + }) + } +} + +func TestScheduler_ConcurrentReplacements(t *testing.T) { + s := NewScheduler(nil, nil, nil) + var wg sync.WaitGroup + for i := range 20 { + wg.Add(1) + go func() { + defer wg.Done() + schedule := "0 0 * * * *" + if i%2 == 0 { + schedule = "invalid" + } + ds := &types.DataSource{ID: "concurrent", Status: types.DataSourceStatusActive, SyncSchedule: schedule} + if err := s.AddOrUpdate(ds); (err != nil) != (i%2 == 0) { + t.Errorf("unexpected result for schedule %q: %v", schedule, err) + } + }() + } + wg.Wait() + if got := s.EntryCount(); got != 1 { + t.Fatalf("concurrent replacements left %d entries, want 1", got) + } +} + +func TestScheduler_PauseRemovesEntryWithInvalidStoredSchedule(t *testing.T) { + s := NewScheduler(nil, nil, nil) + ds := &types.DataSource{ID: "pause", Status: types.DataSourceStatusActive, SyncSchedule: "0 0 * * * *"} + if err := s.AddOrUpdate(ds); err != nil { + t.Fatal(err) + } + ds.Status = types.DataSourceStatusPaused + ds.SyncSchedule = "invalid" + if err := s.AddOrUpdate(ds); err != nil { + t.Fatal(err) + } + if got := s.EntryCount(); got != 0 { + t.Fatalf("pause left %d entries", got) + } +} + func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) { repo := newFakeDataSourceRepo() // Create a data source that is paused diff --git a/internal/handler/datasource_schedule_test.go b/internal/handler/datasource_schedule_test.go new file mode 100644 index 00000000..f03fcb23 --- /dev/null +++ b/internal/handler/datasource_schedule_test.go @@ -0,0 +1,80 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestDataSourceScheduleAPIRejectsInvalidCron(t *testing.T) { + for _, operation := range []string{"create", "update", "resume"} { + t.Run(operation, func(t *testing.T) { + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "schedule.db")), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.DataSource{})) + repo := repository.NewDataSourceRepository(db) + scheduler := datasource.NewScheduler(repo, nil, nil) + kbSvc := &stubKBServiceForDS{getByID: func(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: "kb-schedule", TenantID: 1}, nil + }} + svc := service.NewDataSourceService(repo, nil, nil, kbSvc, nil, nil, scheduler, nil, nil, nil) + ds := &types.DataSource{ + ID: "ds-schedule", KnowledgeBaseID: "kb-schedule", TenantID: 1, + Type: types.ConnectorTypeRSS, Status: types.DataSourceStatusActive, + SyncSchedule: "0 0 * * * *", + } + if operation == "resume" { + ds.Status = types.DataSourceStatusPaused + ds.SyncSchedule = "invalid" + } + if operation != "create" { + require.NoError(t, repo.Create(context.Background(), ds)) + require.NoError(t, scheduler.AddOrUpdate(ds)) + } + entries := scheduler.EntryCount() + h := NewDataSourceHandler(svc, kbSvc) + router := newDataSourceTestRouter(h) + router.POST("/datasource", h.CreateDataSource) + router.PUT("/datasource/:id", h.UpdateDataSource) + router.POST("/datasource/:id/resume", h.ResumeDataSource) + body := *ds + body.SyncSchedule = "invalid" + payload, err := json.Marshal(body) + require.NoError(t, err) + method, url := http.MethodPost, "/datasource" + switch operation { + case "update": + method, url = http.MethodPut, "/datasource/"+ds.ID + case "resume": + url = "/datasource/" + ds.ID + "/resume" + } + req := withDSCtx(httptest.NewRequest(method, url, bytes.NewReader(payload)), 1) + req.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + require.Equal(t, http.StatusBadRequest, response.Code, response.Body.String()) + require.Contains(t, response.Body.String(), "invalid cron expression") + require.Equal(t, entries, scheduler.EntryCount()) + saved, err := svc.GetDataSource(context.Background(), ds.ID) + if operation == "create" { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, ds.Status, saved.Status) + require.Equal(t, ds.SyncSchedule, saved.SyncSchedule) + } + }) + } +} diff --git a/website-docs/03-features/10-datasource.md b/website-docs/03-features/10-datasource.md index 9439ca20..8441c6df 100644 --- a/website-docs/03-features/10-datasource.md +++ b/website-docs/03-features/10-datasource.md @@ -20,6 +20,8 @@ 凭据更新需要提交完整配置,系统会在线验证。暂停数据源可停止后续计划同步;恢复后重新注册调度。 +`sync_schedule` 使用包含秒的六段 cron,例如 `0 0 */6 * * *` 表示每六小时触发;空字符串表示仅手动同步。创建、编辑及恢复数据源时会校验表达式,无效值返回 HTTP 400,不改变已保存配置或已有定时任务。历史数据源若保存了无效表达式,需先修改为合法表达式或清空,再恢复同步。 + ## 选择连接器 | 连接器 | 类型标识 | 同步对象 | diff --git a/website-docs/04-api/02-api-infra.md b/website-docs/04-api/02-api-infra.md index 847c9687..9c61c699 100644 --- a/website-docs/04-api/02-api-infra.md +++ b/website-docs/04-api/02-api-infra.md @@ -374,11 +374,17 @@ curl $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN" 用途:更新(`id/tenant_id/knowledge_base_id` 锁定为原值)。权限:Admin+。请求体同创建。 +PUT 整体替换数据源配置,不支持部分更新:请先 GET 详情,修改后回传完整对象(`name`、`type`、`config`、`sync_schedule`、`sync_mode`、`conflict_strategy`、`sync_deletions`、`sync_log_retention_days`、`status` 等)。`sync_schedule` 和 `sync_deletions` 总是按请求体写入:省略 `sync_schedule` 会清空为空字符串(仅手动同步,已有定时任务随之移除),省略 `sync_deletions` 即为 false;其他字段省略时是否保留原值不作保证。凭证不经本接口修改,请求体中的 `config.credentials` 会被忽略,保留已存凭证。 + 响应:200 `DataSourceResponse` ```bash curl -X PUT $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN" \ - -H 'Content-Type: application/json' -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1","config":{}}' + -H 'Content-Type: application/json' \ + -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1", + "config":{"type":"notion","resource_ids":["page-1"]}, + "sync_schedule":"0 0 */6 * * *","sync_mode":"incremental","conflict_strategy":"overwrite", + "sync_deletions":true,"sync_log_retention_days":30,"status":"active"}' ``` ### DELETE /api/v1/datasource/:id