fix(datasource): validate cron schedules and persist manual-only mode (#3653)
* fix(datasource): preserve schedule state on invalid updates * docs(datasource): document PUT /datasource/:id as a full replace The update now always persists sync_schedule, so a partial body clears the schedule. Show a full-object example and state that omitted fields are reset rather than preserved. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>main
parent
a9d71038b6
commit
00e6366fd2
|
|
@ -90,10 +90,13 @@ func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource)
|
|||
return err
|
||||
}
|
||||
// GORM Updates(struct) deliberately skips zero values, which would make
|
||||
// a user-selected sync_deletions=false impossible to persist.
|
||||
// a user-selected sync_deletions=false or manual-only schedule impossible to persist.
|
||||
return tx.Model(&types.DataSource{}).
|
||||
Where("id = ?", ds.ID).
|
||||
UpdateColumn("sync_deletions", ds.SyncDeletions).Error
|
||||
UpdateColumns(map[string]interface{}{
|
||||
"sync_deletions": ds.SyncDeletions,
|
||||
"sync_schedule": ds.SyncSchedule,
|
||||
}).Error
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDataSourceResumeRejectsInvalidSchedule(t *testing.T) {
|
||||
f := newSQLiteDataSourceDeleteFixture(t)
|
||||
f.scheduler.Remove(f.ds.ID)
|
||||
f.ds.Status = types.DataSourceStatusPaused
|
||||
f.ds.SyncSchedule = "not-a-cron"
|
||||
require.NoError(t, f.dsRepo.Update(context.Background(), f.ds))
|
||||
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
|
||||
err := svc.ResumeDataSource(context.Background(), f.ds.ID)
|
||||
saved, readErr := svc.GetDataSource(context.Background(), f.ds.ID)
|
||||
require.NoError(t, readErr)
|
||||
t.Logf("resume error=%v; status=%s; entries=%d", err, saved.Status, f.scheduler.EntryCount())
|
||||
require.ErrorContains(t, err, "invalid cron expression")
|
||||
require.Equal(t, types.DataSourceStatusPaused, saved.Status)
|
||||
require.Zero(t, f.scheduler.EntryCount())
|
||||
}
|
||||
|
||||
func TestDataSourceUpdateRejectsInvalidSchedule(t *testing.T) {
|
||||
f := newSQLiteDataSourceDeleteFixture(t)
|
||||
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
|
||||
updated := *f.ds
|
||||
updated.SyncSchedule = "not-a-cron"
|
||||
_, err := svc.UpdateDataSource(context.Background(), &updated)
|
||||
require.ErrorContains(t, err, "invalid cron expression")
|
||||
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)
|
||||
require.Equal(t, 1, f.scheduler.EntryCount())
|
||||
}
|
||||
|
||||
func TestDataSourceCreateRejectsInvalidSchedule(t *testing.T) {
|
||||
f := newSQLiteDataSourceDeleteFixture(t)
|
||||
registry := datasource.NewConnectorRegistry()
|
||||
require.NoError(t, registry.Register(deletedItemConnector{}))
|
||||
svc := &DataSourceService{
|
||||
dsRepo: f.dsRepo, scheduler: f.scheduler, connectorRegistry: registry,
|
||||
kbService: &processSyncKBService{kb: &types.KnowledgeBase{ID: f.ds.KnowledgeBaseID, TenantID: f.ds.TenantID}},
|
||||
}
|
||||
ds := *f.ds
|
||||
ds.ID = "invalid-schedule"
|
||||
ds.Type = deletedItemConnectorType
|
||||
ds.SyncSchedule = "not-a-cron"
|
||||
ds.Config = []byte(`{"type":"test"}`)
|
||||
_, err := svc.CreateDataSource(context.Background(), &ds)
|
||||
require.ErrorContains(t, err, "invalid cron expression")
|
||||
_, err = svc.GetDataSource(context.Background(), ds.ID)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, f.scheduler.EntryCount())
|
||||
}
|
||||
|
||||
func TestDataSourceScheduleLifecycle(t *testing.T) {
|
||||
for _, schedule := range []string{"", "0 0 */6 * * *", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {
|
||||
t.Run(schedule, func(t *testing.T) {
|
||||
f := newSQLiteDataSourceDeleteFixture(t)
|
||||
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
|
||||
updated := *f.ds
|
||||
updated.SyncSchedule = schedule
|
||||
_, err := svc.UpdateDataSource(context.Background(), &updated)
|
||||
require.NoError(t, err)
|
||||
wantEntries := 1
|
||||
if schedule == "" {
|
||||
wantEntries = 0
|
||||
}
|
||||
require.Equal(t, wantEntries, f.scheduler.EntryCount())
|
||||
require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))
|
||||
require.Zero(t, f.scheduler.EntryCount())
|
||||
// Repeated resume must replace, rather than duplicate, the job.
|
||||
for range 2 {
|
||||
require.NoError(t, svc.ResumeDataSource(context.Background(), f.ds.ID))
|
||||
require.Equal(t, wantEntries, f.scheduler.EntryCount())
|
||||
}
|
||||
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, types.DataSourceStatusActive, saved.Status)
|
||||
require.Equal(t, schedule, saved.SyncSchedule)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataSourceScheduleWriteFailurePreservesState(t *testing.T) {
|
||||
for _, operation := range []string{"update", "resume"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
f := newSQLiteDataSourceDeleteFixture(t)
|
||||
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
|
||||
if operation == "resume" {
|
||||
require.NoError(t, svc.PauseDataSource(context.Background(), f.ds.ID))
|
||||
}
|
||||
before, err := svc.GetDataSource(context.Background(), f.ds.ID)
|
||||
require.NoError(t, err)
|
||||
entries := f.scheduler.EntryCount()
|
||||
require.NoError(t, f.db.Exec(`CREATE TRIGGER fail_schedule_write BEFORE UPDATE ON data_sources
|
||||
BEGIN SELECT RAISE(FAIL, 'forced schedule write failure'); END;`).Error)
|
||||
if operation == "resume" {
|
||||
err = svc.ResumeDataSource(context.Background(), f.ds.ID)
|
||||
} else {
|
||||
updated := *before
|
||||
updated.SyncSchedule = "0 30 * * * *"
|
||||
_, err = svc.UpdateDataSource(context.Background(), &updated)
|
||||
}
|
||||
require.ErrorContains(t, err, "forced schedule write failure")
|
||||
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, before.Status, saved.Status)
|
||||
require.Equal(t, before.SyncSchedule, saved.SyncSchedule)
|
||||
require.Equal(t, entries, f.scheduler.EntryCount())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataSourceScheduleCanceledUpdatePreservesState(t *testing.T) {
|
||||
f := newSQLiteDataSourceDeleteFixture(t)
|
||||
svc := &DataSourceService{dsRepo: f.dsRepo, scheduler: f.scheduler}
|
||||
updated := *f.ds
|
||||
updated.SyncSchedule = "0 30 * * * *"
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := svc.UpdateDataSource(ctx, &updated)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
saved, err := svc.GetDataSource(context.Background(), f.ds.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, f.ds.SyncSchedule, saved.SyncSchedule)
|
||||
require.Equal(t, 1, f.scheduler.EntryCount())
|
||||
}
|
||||
|
|
@ -69,6 +69,9 @@ func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.Data
|
|||
if ds == nil {
|
||||
return nil, datasource.ErrDataSourceInvalid
|
||||
}
|
||||
if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate knowledge base exists
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID)
|
||||
|
|
@ -169,6 +172,9 @@ func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.Data
|
|||
if ds.TenantID != existing.TenantID {
|
||||
return nil, datasource.ErrDataSourceInvalid
|
||||
}
|
||||
if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Credentials NEVER flow through this endpoint — they live behind the
|
||||
// /credentials subresource. Force-preserve the stored credentials map
|
||||
|
|
@ -549,6 +555,9 @@ func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) err
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := datasource.ValidateSyncSchedule(ds.SyncSchedule); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ds.Status = types.DataSourceStatusActive
|
||||
if err := s.dsRepo.Update(ctx, ds); err != nil {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,22 @@ import (
|
|||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
var syncScheduleParser = cron.NewParser(
|
||||
cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
|
||||
)
|
||||
|
||||
// ValidateSyncSchedule accepts the same syntax as the scheduler. An empty
|
||||
// schedule disables automatic sync and remains valid for manual-only sources.
|
||||
func ValidateSyncSchedule(schedule string) error {
|
||||
if schedule == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := syncScheduleParser.Parse(schedule); err != nil {
|
||||
return fmt.Errorf("invalid cron expression %q: %w", schedule, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Scheduler manages cron-based periodic sync for data sources.
|
||||
//
|
||||
// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires
|
||||
|
|
@ -41,7 +57,7 @@ func NewScheduler(
|
|||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
) *Scheduler {
|
||||
return &Scheduler{
|
||||
cron: cron.New(cron.WithSeconds(), cron.WithChain(
|
||||
cron: cron.New(cron.WithParser(syncScheduleParser), cron.WithChain(
|
||||
cron.Recover(cron.DefaultLogger),
|
||||
)),
|
||||
dsRepo: dsRepo,
|
||||
|
|
@ -82,6 +98,13 @@ func (s *Scheduler) Stop() {
|
|||
|
||||
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
|
||||
func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error {
|
||||
// Reject an invalid replacement before removing the working entry. Inactive
|
||||
// sources must still be removable even if their stored schedule is invalid.
|
||||
if ds.Status == types.DataSourceStatusActive {
|
||||
if err := ValidateSyncSchedule(ds.SyncSchedule); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
|
|
|
|||
|
|
@ -345,6 +345,92 @@ func TestScheduler_InvalidCron(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestScheduler_InvalidReplacementPreservesSchedule(t *testing.T) {
|
||||
repo := newFakeDataSourceRepo()
|
||||
enqueuer := &fakeTaskEnqueuer{}
|
||||
s := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer)
|
||||
ds := &types.DataSource{ID: "replace", Status: types.DataSourceStatusActive, SyncSchedule: "* * * * * *"}
|
||||
if err := repo.Create(context.Background(), ds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AddOrUpdate(ds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
invalid := *ds
|
||||
invalid.SyncSchedule = "not-a-cron"
|
||||
if err := s.AddOrUpdate(&invalid); err == nil {
|
||||
t.Fatal("expected invalid schedule to be rejected")
|
||||
}
|
||||
if got := s.EntryCount(); got != 1 {
|
||||
t.Fatalf("invalid replacement removed existing schedule: entries=%d", got)
|
||||
}
|
||||
s.cron.Start()
|
||||
defer s.Stop()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for enqueuer.count.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if enqueuer.count.Load() == 0 {
|
||||
t.Fatal("preserved schedule did not enqueue a task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSyncSchedule(t *testing.T) {
|
||||
for _, schedule := range []string{"", "0 0 */6 * * *", "@hourly", "@every 6h", "CRON_TZ=UTC 0 0 * * * *"} {
|
||||
t.Run(schedule, func(t *testing.T) {
|
||||
if err := ValidateSyncSchedule(schedule); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, schedule := range []string{"not-a-cron", "0 0 * * *", "60 * * * * *", "CRON_TZ=Invalid/Zone 0 0 * * * *"} {
|
||||
t.Run(schedule, func(t *testing.T) {
|
||||
if err := ValidateSyncSchedule(schedule); err == nil {
|
||||
t.Fatal("expected invalid schedule to be rejected")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_ConcurrentReplacements(t *testing.T) {
|
||||
s := NewScheduler(nil, nil, nil)
|
||||
var wg sync.WaitGroup
|
||||
for i := range 20 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
schedule := "0 0 * * * *"
|
||||
if i%2 == 0 {
|
||||
schedule = "invalid"
|
||||
}
|
||||
ds := &types.DataSource{ID: "concurrent", Status: types.DataSourceStatusActive, SyncSchedule: schedule}
|
||||
if err := s.AddOrUpdate(ds); (err != nil) != (i%2 == 0) {
|
||||
t.Errorf("unexpected result for schedule %q: %v", schedule, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if got := s.EntryCount(); got != 1 {
|
||||
t.Fatalf("concurrent replacements left %d entries, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_PauseRemovesEntryWithInvalidStoredSchedule(t *testing.T) {
|
||||
s := NewScheduler(nil, nil, nil)
|
||||
ds := &types.DataSource{ID: "pause", Status: types.DataSourceStatusActive, SyncSchedule: "0 0 * * * *"}
|
||||
if err := s.AddOrUpdate(ds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds.Status = types.DataSourceStatusPaused
|
||||
ds.SyncSchedule = "invalid"
|
||||
if err := s.AddOrUpdate(ds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := s.EntryCount(); got != 0 {
|
||||
t.Fatalf("pause left %d entries", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) {
|
||||
repo := newFakeDataSourceRepo()
|
||||
// Create a data source that is paused
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestDataSourceScheduleAPIRejectsInvalidCron(t *testing.T) {
|
||||
for _, operation := range []string{"create", "update", "resume"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "schedule.db")), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&types.DataSource{}))
|
||||
repo := repository.NewDataSourceRepository(db)
|
||||
scheduler := datasource.NewScheduler(repo, nil, nil)
|
||||
kbSvc := &stubKBServiceForDS{getByID: func(context.Context, string) (*types.KnowledgeBase, error) {
|
||||
return &types.KnowledgeBase{ID: "kb-schedule", TenantID: 1}, nil
|
||||
}}
|
||||
svc := service.NewDataSourceService(repo, nil, nil, kbSvc, nil, nil, scheduler, nil, nil, nil)
|
||||
ds := &types.DataSource{
|
||||
ID: "ds-schedule", KnowledgeBaseID: "kb-schedule", TenantID: 1,
|
||||
Type: types.ConnectorTypeRSS, Status: types.DataSourceStatusActive,
|
||||
SyncSchedule: "0 0 * * * *",
|
||||
}
|
||||
if operation == "resume" {
|
||||
ds.Status = types.DataSourceStatusPaused
|
||||
ds.SyncSchedule = "invalid"
|
||||
}
|
||||
if operation != "create" {
|
||||
require.NoError(t, repo.Create(context.Background(), ds))
|
||||
require.NoError(t, scheduler.AddOrUpdate(ds))
|
||||
}
|
||||
entries := scheduler.EntryCount()
|
||||
h := NewDataSourceHandler(svc, kbSvc)
|
||||
router := newDataSourceTestRouter(h)
|
||||
router.POST("/datasource", h.CreateDataSource)
|
||||
router.PUT("/datasource/:id", h.UpdateDataSource)
|
||||
router.POST("/datasource/:id/resume", h.ResumeDataSource)
|
||||
body := *ds
|
||||
body.SyncSchedule = "invalid"
|
||||
payload, err := json.Marshal(body)
|
||||
require.NoError(t, err)
|
||||
method, url := http.MethodPost, "/datasource"
|
||||
switch operation {
|
||||
case "update":
|
||||
method, url = http.MethodPut, "/datasource/"+ds.ID
|
||||
case "resume":
|
||||
url = "/datasource/" + ds.ID + "/resume"
|
||||
}
|
||||
req := withDSCtx(httptest.NewRequest(method, url, bytes.NewReader(payload)), 1)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
router.ServeHTTP(response, req)
|
||||
require.Equal(t, http.StatusBadRequest, response.Code, response.Body.String())
|
||||
require.Contains(t, response.Body.String(), "invalid cron expression")
|
||||
require.Equal(t, entries, scheduler.EntryCount())
|
||||
saved, err := svc.GetDataSource(context.Background(), ds.ID)
|
||||
if operation == "create" {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, ds.Status, saved.Status)
|
||||
require.Equal(t, ds.SyncSchedule, saved.SyncSchedule)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,8 @@
|
|||
|
||||
凭据更新需要提交完整配置,系统会在线验证。暂停数据源可停止后续计划同步;恢复后重新注册调度。
|
||||
|
||||
`sync_schedule` 使用包含秒的六段 cron,例如 `0 0 */6 * * *` 表示每六小时触发;空字符串表示仅手动同步。创建、编辑及恢复数据源时会校验表达式,无效值返回 HTTP 400,不改变已保存配置或已有定时任务。历史数据源若保存了无效表达式,需先修改为合法表达式或清空,再恢复同步。
|
||||
|
||||
## 选择连接器
|
||||
|
||||
| 连接器 | 类型标识 | 同步对象 |
|
||||
|
|
|
|||
|
|
@ -374,11 +374,17 @@ curl $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN"
|
|||
|
||||
用途:更新(`id/tenant_id/knowledge_base_id` 锁定为原值)。权限:Admin+。请求体同创建。
|
||||
|
||||
PUT 整体替换数据源配置,不支持部分更新:请先 GET 详情,修改后回传完整对象(`name`、`type`、`config`、`sync_schedule`、`sync_mode`、`conflict_strategy`、`sync_deletions`、`sync_log_retention_days`、`status` 等)。`sync_schedule` 和 `sync_deletions` 总是按请求体写入:省略 `sync_schedule` 会清空为空字符串(仅手动同步,已有定时任务随之移除),省略 `sync_deletions` 即为 false;其他字段省略时是否保留原值不作保证。凭证不经本接口修改,请求体中的 `config.credentials` 会被忽略,保留已存凭证。
|
||||
|
||||
响应:200 `DataSourceResponse`
|
||||
|
||||
```bash
|
||||
curl -X PUT $BASE/api/v1/datasource/ds-1 -H "Authorization: Bearer $TOKEN" \
|
||||
-H 'Content-Type: application/json' -d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1","config":{}}'
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"name":"notion 同步 v2","type":"notion","knowledge_base_id":"kb-1",
|
||||
"config":{"type":"notion","resource_ids":["page-1"]},
|
||||
"sync_schedule":"0 0 */6 * * *","sync_mode":"incremental","conflict_strategy":"overwrite",
|
||||
"sync_deletions":true,"sync_log_retention_days":30,"status":"active"}'
|
||||
```
|
||||
|
||||
### DELETE /api/v1/datasource/:id
|
||||
|
|
|
|||
Loading…
Reference in New Issue