diff --git a/.github/workflows/go-lint.yml b/.github/workflows/go-lint.yml index af10de96..c95d3ec6 100644 --- a/.github/workflows/go-lint.yml +++ b/.github/workflows/go-lint.yml @@ -58,10 +58,10 @@ jobs: modules=() if git diff --name-only --diff-filter=ACMR "$diff_range" -- \ .golangci.yml .github/workflows/go-lint.yml | grep -q .; then - modules=("." "cli" "client") + modules=("." "cli" "client" "pluginsdk") else if git diff --name-only --diff-filter=ACMR "$diff_range" -- \ - '*.go' ':!cli/**' ':!client/**' go.mod go.sum | grep -q .; then + '*.go' ':!cli/**' ':!client/**' ':!pluginsdk/**' go.mod go.sum | grep -q .; then modules+=(".") fi if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'cli/**' | grep -q .; then @@ -70,6 +70,9 @@ jobs: if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'client/**' | grep -q .; then modules+=("client") fi + if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'pluginsdk/**' | grep -q .; then + modules+=("pluginsdk") + fi fi json="[" first=1 diff --git a/.github/workflows/plugin-sdk.yml b/.github/workflows/plugin-sdk.yml new file mode 100644 index 00000000..f239bd08 --- /dev/null +++ b/.github/workflows/plugin-sdk.yml @@ -0,0 +1,44 @@ +name: Plugin SDK + +# Builds and tests the pluginsdk/ Go module (extension protocol, Go SDK, +# client and conformance suite) on every push / PR that touches it. + +on: + push: + branches: [main] + paths: + - 'pluginsdk/**' + - '.github/workflows/plugin-sdk.yml' + pull_request: + paths: + - 'pluginsdk/**' + - '.github/workflows/plugin-sdk.yml' + +defaults: + run: + working-directory: pluginsdk + shell: bash + +permissions: + contents: read + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version: '1.26' + cache-dependency-path: pluginsdk/go.mod + - name: go build + run: go build ./... + - name: go test + run: go test -race ./... + - name: go vet + run: go vet ./... diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 21a27292..0900cd47 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -38,10 +38,12 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \ # Install migrate tool RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest -# Copy go mod files. go.mod replace-points anydoc at ./third_party/anydoc-go, -# so that module's go.mod must exist before `go mod download`. +# Copy go mod files. go.mod replace-points anydoc at ./third_party/anydoc-go +# and the plugin SDK at ./pluginsdk, so their go.mod files must exist before +# `go mod download`. COPY go.mod go.sum ./ COPY third_party/anydoc-go/go.mod third_party/anydoc-go/go.mod +COPY pluginsdk/go.mod pluginsdk/go.mod RUN --mount=type=cache,target=/go/pkg/mod go mod download COPY cmd/download cmd/download RUN go run cmd/download/duckdb/duckdb.go diff --git a/examples/plugins/rss/feeds.go b/examples/plugins/rss/feeds.go new file mode 100644 index 00000000..a01e1aac --- /dev/null +++ b/examples/plugins/rss/feeds.go @@ -0,0 +1,282 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "strings" + "time" + "unicode/utf8" + + readability "codeberg.org/readeck/go-readability/v2" + htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2" + "github.com/mmcdole/gofeed" +) + +const ( + requestTimeout = 20 * time.Second + maxFeedSize = 10 << 20 + maxArticleSize = 5 << 20 + defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)" + maxFileNameBytes = 200 +) + +// config is the data source configuration: feed URLs in settings, optional +// auth headers (which may carry secrets) in credentials. +type config struct { + feedURLs []string + authHeaders map[string]string +} + +func parseConfig(settings, credentials map[string]any) (*config, error) { + raw, _ := settings["feed_urls"].(string) + if strings.TrimSpace(raw) == "" { + // Older rows kept the URLs with the credentials. + raw, _ = credentials["feed_urls"].(string) + } + cfg := &config{feedURLs: splitURLs(raw)} + if len(cfg.feedURLs) == 0 { + return nil, errMissingFeeds + } + headers, _ := credentials["auth_headers"].(string) + cfg.authHeaders = parseHeaders(headers) + return cfg, nil +} + +// splitURLs splits on newlines and commas, trims, dedupes (order kept). +func splitURLs(raw string) []string { + parts := strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' }) + seen := map[string]bool{} + var out []string + for _, u := range parts { + u = strings.TrimSpace(u) + if u == "" || seen[u] { + continue + } + seen[u] = true + out = append(out, u) + } + return out +} + +func parseHeaders(raw string) map[string]string { + headers := map[string]string{} + for _, line := range strings.Split(raw, "\n") { + name, value, ok := strings.Cut(strings.TrimSpace(line), ":") + name = strings.TrimSpace(name) + if !ok || name == "" { + continue + } + headers[name] = strings.TrimSpace(value) + } + if len(headers) == 0 { + return nil + } + return headers +} + +// fetcher downloads feeds and article pages. Outbound traffic goes through +// the plugin host's egress proxy (HTTPS_PROXY), which refuses private +// addresses, so the plugin needs no SSRF logic of its own. +type fetcher struct { + http *http.Client + headers map[string]string +} + +func newFetcher(headers map[string]string) *fetcher { + return &fetcher{http: &http.Client{Timeout: requestTimeout}, headers: headers} +} + +// fetch attaches auth headers only to feed fetches; article pages on +// third-party sites must not receive feed credentials. +func (f *fetcher) fetch(ctx context.Context, rawURL string, maxSize int64, withAuth bool) ([]byte, error) { + u, err := url.Parse(rawURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") { + return nil, fmt.Errorf("invalid URL %q", rawURL) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + if withAuth { + for k, v := range f.headers { + req.Header.Set(k, v) + } + } + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", defaultUserAgent) + } + if req.Header.Get("Accept") == "" { + req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, "+ + "application/json, text/html;q=0.9, */*;q=0.8") + } + resp, err := f.http.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1)) + if err != nil { + return nil, fmt.Errorf("read body failed: %w", err) + } + if int64(len(body)) > maxSize { + return nil, fmt.Errorf("response exceeds maximum size (%d bytes)", maxSize) + } + return body, nil +} + +func (f *fetcher) feed(ctx context.Context, feedURL string) (*gofeed.Feed, error) { + data, err := f.fetch(ctx, feedURL, maxFeedSize, true) + if err != nil { + return nil, err + } + feed, err := gofeed.NewParser().Parse(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("parse feed: %w", err) + } + return feed, nil +} + +// article returns the readability-cleaned main content of a page as HTML. +func (f *fetcher) article(ctx context.Context, articleURL string) (string, string, error) { + body, err := f.fetch(ctx, articleURL, maxArticleSize, false) + if err != nil { + return "", "", err + } + pageURL, _ := url.Parse(articleURL) + art, err := readability.FromReader(bytes.NewReader(body), pageURL) + if err != nil { + return "", "", fmt.Errorf("readability parse: %w", err) + } + if art.Node == nil { + return "", "", fmt.Errorf("no readable content extracted") + } + var buf bytes.Buffer + if err := art.RenderHTML(&buf); err != nil { + return "", "", fmt.Errorf("render article html: %w", err) + } + return buf.String(), art.Title(), nil +} + +// cursorState is the incremental sync state, the same shape the builtin +// connector keeps: feed URL → item ID → content fingerprint, and feed URL → +// item ID → feed-only signal that lets unchanged entries skip article fetches. +type cursorState struct { + FeedItems map[string]map[string]string + FeedSignals map[string]map[string]string +} + +func newCursorState() *cursorState { + return &cursorState{FeedItems: map[string]map[string]string{}, FeedSignals: map[string]map[string]string{}} +} + +func cursorFromState(state map[string]any) *cursorState { + c := newCursorState() + read := func(key string, into map[string]map[string]string) { + feeds, _ := state[key].(map[string]any) + for feedURL, items := range feeds { + m, _ := items.(map[string]any) + into[feedURL] = map[string]string{} + for id, v := range m { + if s, ok := v.(string); ok { + into[feedURL][id] = s + } + } + } + } + read("feed_items", c.FeedItems) + read("feed_signals", c.FeedSignals) + return c +} + +func (c *cursorState) toState() map[string]any { + return map[string]any{"feed_items": c.FeedItems, "feed_signals": c.FeedSignals} +} + +func (c *cursorState) copyFeed(prev *cursorState, feedURL string) { + if prev == nil { + return + } + if src := prev.FeedItems[feedURL]; len(src) > 0 { + c.FeedItems[feedURL] = maps.Clone(src) + } + if src := prev.FeedSignals[feedURL]; len(src) > 0 { + c.FeedSignals[feedURL] = maps.Clone(src) + } +} + +func contentFingerprint(markdown string) string { + sum := sha256.Sum256([]byte(markdown)) + return "h:" + hex.EncodeToString(sum[:])[:16] +} + +func feedSignal(item *gofeed.Item, feedContent string) string { + var b strings.Builder + b.WriteString(item.GUID + "\n" + item.Link + "\n" + item.Title + "\n") + if item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero() { + b.WriteString(item.UpdatedParsed.UTC().Format(time.RFC3339)) + } + b.WriteByte('\n') + if item.PublishedParsed != nil && !item.PublishedParsed.IsZero() { + b.WriteString(item.PublishedParsed.UTC().Format(time.RFC3339)) + } + b.WriteByte('\n') + b.WriteString(feedContent) + sum := sha256.Sum256([]byte(b.String())) + return "s:" + hex.EncodeToString(sum[:])[:16] +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +func htmlToMarkdown(html string) string { + if strings.TrimSpace(html) == "" { + return "" + } + md, err := htmltomd.ConvertString(html) + if err != nil || strings.TrimSpace(md) == "" { + return strings.TrimSpace(html) + } + return strings.TrimSpace(md) +} + +var fileNameReplacer = strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", +) + +// fileName makes a feed title safe to use as a file name, the way WeKnora's +// builtin connectors do: hostile punctuation replaced, at most 200 bytes +// without splitting a rune. +func fileName(title string) string { + title = strings.TrimSpace(strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(title)) + if title == "" { + return "untitled.md" + } + name := fileNameReplacer.Replace(title) + if len(name) > maxFileNameBytes { + name = name[:maxFileNameBytes] + for len(name) > 0 { + r, size := utf8.DecodeLastRuneInString(name) + if r != utf8.RuneError || size != 1 { + break + } + name = name[:len(name)-1] + } + } + return name + ".md" +} diff --git a/examples/plugins/rss/icon.svg b/examples/plugins/rss/icon.svg new file mode 100644 index 00000000..485e6cec --- /dev/null +++ b/examples/plugins/rss/icon.svg @@ -0,0 +1 @@ + diff --git a/examples/plugins/rss/main.go b/examples/plugins/rss/main.go new file mode 100644 index 00000000..d1aaa55e --- /dev/null +++ b/examples/plugins/rss/main.go @@ -0,0 +1,229 @@ +// Command rss is WeKnora's RSS/Atom/JSON feed connector written as an +// external plugin with the Go SDK. It behaves like the builtin rss connector +// (a test syncs the same feeds through both) and shows how a code plugin is +// built: it imports only the SDK and ordinary libraries, never WeKnora's +// internals. +// +// Build a package with ./package.sh; install it under System administration +// → Plugin management. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/mmcdole/gofeed" + + "github.com/Tencent/WeKnora/pluginsdk" + "github.com/Tencent/WeKnora/pluginsdk/pluginapi" +) + +// Version must match plugin.yaml. +const Version = "1.0.0" + +var errMissingFeeds = pluginapi.InvalidConfig( + "feed_urls is required", + map[string]string{"settings.feed_urls": "required"}, +) + +func main() { + p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version}) + p.Connector("rss", &connector{logf: p.Logger().Warn}) + if err := p.Serve(); err != nil { + log.Fatal(err) + } +} + +type connector struct { + logf func(msg string, args ...any) +} + +func (c *connector) warn(msg string, args ...any) { + if c.logf != nil { + c.logf(msg, args...) + } +} + +// Validate fetches and parses every configured feed. +func (c *connector) Validate(ctx context.Context, _ *pluginsdk.Call, in pluginsdk.ConnectorConfig) error { + cfg, err := parseConfig(in.Settings, in.Credentials) + if err != nil { + return err + } + f := newFetcher(cfg.authHeaders) + for _, u := range cfg.feedURLs { + if _, err := f.feed(ctx, u); err != nil { + return pluginapi.InvalidConfig(fmt.Sprintf("feed %s: %v", u, err), + map[string]string{"settings.feed_urls": err.Error()}) + } + } + return nil +} + +// ListResources lists one resource per feed. A feed that fails still +// appears, named by its URL, so the user can deselect it. +func (c *connector) ListResources( + ctx context.Context, _ *pluginsdk.Call, in pluginsdk.ConnectorConfig, parentID string, +) ([]pluginapi.Resource, error) { + if parentID != "" { + return nil, nil + } + cfg, err := parseConfig(in.Settings, in.Credentials) + if err != nil { + return nil, err + } + f := newFetcher(cfg.authHeaders) + out := make([]pluginapi.Resource, 0, len(cfg.feedURLs)) + for _, u := range cfg.feedURLs { + res := pluginapi.Resource{ExternalID: u, Type: "feed", Name: u, URL: u} + feed, err := f.feed(ctx, u) + if err != nil { + res.Description = "fetch failed: " + err.Error() + out = append(out, res) + continue + } + if t := strings.TrimSpace(feed.Title); t != "" { + res.Name = t + } + res.Description = strings.TrimSpace(feed.Description) + if feed.Link != "" { + res.URL = feed.Link + } + res.ModifiedAt = feed.UpdatedParsed + res.Metadata = map[string]any{"item_count": len(feed.Items)} + out = append(out, res) + } + return out, nil +} + +// Fetch syncs the selected feeds (all configured ones when none are +// selected). Incremental runs skip entries whose feed signal and content are +// unchanged; deletions are never emitted, since feeds drop old items as a +// matter of course. +func (c *connector) Fetch( + ctx context.Context, + _ *pluginsdk.Call, + in pluginsdk.ConnectorConfig, + req pluginapi.FetchInput, + s *pluginsdk.Stream, +) (*pluginapi.Cursor, error) { + cfg, err := parseConfig(in.Settings, in.Credentials) + if err != nil { + return nil, err + } + feedURLs := req.ResourceIDs + if len(feedURLs) == 0 { + feedURLs = cfg.feedURLs + } + var prev *cursorState + incremental := req.Mode == pluginapi.FetchIncremental && req.Cursor != nil + if incremental { + prev = cursorFromState(req.Cursor.State) + } + f := newFetcher(cfg.authHeaders) + next := newCursorState() + now := time.Now().UTC() + var failures []string + + for _, feedURL := range feedURLs { + feed, err := f.feed(ctx, feedURL) + if err != nil { + c.warn("feed failed", "url", feedURL, "err", err) + failures = append(failures, fmt.Sprintf("%s: %v", feedURL, err)) + next.copyFeed(prev, feedURL) + continue + } + next.FeedItems[feedURL] = map[string]string{} + next.FeedSignals[feedURL] = map[string]string{} + var prevItems, prevSignals map[string]string + if prev != nil { + prevItems, prevSignals = prev.FeedItems[feedURL], prev.FeedSignals[feedURL] + } + for _, item := range feed.Items { + if item == nil { + continue + } + itemID := firstNonEmpty(item.GUID, item.Link, item.Title) + if itemID == "" { + continue + } + feedContent := firstNonEmpty(item.Content, item.Description) + sig := feedSignal(item, feedContent) + if incremental && prevItems[itemID] != "" && prevSignals[itemID] == sig { + next.FeedItems[feedURL][itemID] = prevItems[itemID] + next.FeedSignals[feedURL][itemID] = sig + continue + } + fetched, fp := c.resolve(ctx, f, feed, item, feedURL, itemID, feedContent) + next.FeedItems[feedURL][itemID] = fp + next.FeedSignals[feedURL][itemID] = sig + if incremental && prevItems[itemID] == fp { + continue + } + if err := s.Item(fetched); err != nil { + return nil, err + } + } + // Each finished feed is a safe place to resume from. + if err := s.Checkpoint(pluginapi.Cursor{LastSyncTime: &now, State: next.toState()}); err != nil { + return nil, err + } + } + if len(failures) > 0 && len(failures) == len(feedURLs) { + return nil, pluginapi.Errorf(pluginapi.CodeUnavailable, "all feeds failed: %s", strings.Join(failures, "; ")) + } + if len(failures) > 0 { + _ = s.Log("warn", "some feeds failed: "+strings.Join(failures, "; ")) + } + return &pluginapi.Cursor{LastSyncTime: &now, State: next.toState()}, nil +} + +// resolve builds one item: the article's full text when it can be fetched, +// the feed's own content otherwise, as Markdown. +func (c *connector) resolve( + ctx context.Context, f *fetcher, feed *gofeed.Feed, item *gofeed.Item, feedURL, itemID, feedContent string, +) (pluginapi.FetchedItem, string) { + title := firstNonEmpty(item.Title, "untitled") + contentHTML := feedContent + if strings.TrimSpace(item.Link) != "" { + html, articleTitle, err := f.article(ctx, item.Link) + if err == nil { + contentHTML = html + if item.Title == "" && articleTitle != "" { + title = articleTitle + } + } else if !errors.Is(err, context.Canceled) { + c.warn("full-text fetch failed, using feed content", "url", item.Link, "err", err) + } + } + content := htmlToMarkdown(contentHTML) + updated := time.Now().UTC() + switch { + case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero(): + updated = *item.UpdatedParsed + case item.PublishedParsed != nil && !item.PublishedParsed.IsZero(): + updated = *item.PublishedParsed + } + author := "" + if item.Author != nil { + author = item.Author.Name + } + return pluginapi.FetchedItem{ + ExternalID: feedURL + ":" + itemID, + Title: title, + Content: []byte(content), + ContentType: "text/markdown", + FileName: fileName(title), + URL: item.Link, + UpdatedAt: &updated, + SourceResourceID: feedURL, + Metadata: map[string]string{ + "channel": "rss", "feed_url": feedURL, "feed_title": feed.Title, + "guid": item.GUID, "link": item.Link, "author": author, + }, + }, contentFingerprint(content) +} diff --git a/examples/plugins/rss/package.sh b/examples/plugins/rss/package.sh new file mode 100755 index 00000000..914d5164 --- /dev/null +++ b/examples/plugins/rss/package.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Builds the RSS example plugin for the platforms WeKnora runs on and zips it +# into a .wkp package: plugin.yaml, schemas, icon and bin/-/rss. +set -euo pipefail +cd "$(dirname "$0")" +version=$(sed -n 's/^version: //p' plugin.yaml) +out=${1:-"weknora-examples-rss-${version}.wkp"} +stage=$(mktemp -d) +trap 'rm -rf "$stage"' EXIT +cp plugin.yaml icon.svg "$stage"/ +cp -r schemas "$stage"/ +for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64; do + os=${target%/*} + arch=${target#*/} + bin="$stage/bin/${os}-${arch}/rss" + [ "$os" = windows ] && bin="$bin.exe" + CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -trimpath -ldflags="-s -w" -o "$bin" . +done +rm -f "$out" +(cd "$stage" && zip -qr - .) > "$out" +echo "wrote $out" diff --git a/examples/plugins/rss/parity_test.go b/examples/plugins/rss/parity_test.go new file mode 100644 index 00000000..d546d92c --- /dev/null +++ b/examples/plugins/rss/parity_test.go @@ -0,0 +1,279 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "go/parser" + "go/token" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + builtin "github.com/Tencent/WeKnora/internal/datasource/connector/rss" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" + "github.com/Tencent/WeKnora/pluginsdk" + "github.com/Tencent/WeKnora/pluginsdk/client" + "github.com/Tencent/WeKnora/pluginsdk/conformance" + "github.com/Tencent/WeKnora/pluginsdk/pluginapi" +) + +// feedSite serves two feeds and the article pages they link to. Changing an +// entry's text simulates a feed update between syncs. +type feedSite struct { + *httptest.Server + mu sync.Mutex + secondA string +} + +func newFeedSite(t *testing.T) *feedSite { + s := &feedSite{secondA: "Second post, first version."} + mux := http.NewServeMux() + mux.HandleFunc("/a.xml", func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + second := s.secondA + s.mu.Unlock() + w.Header().Set("Content-Type", "application/rss+xml") + _, _ = fmt.Fprintf(w, `Feed A%[1]s +First feed +Hello: world%[1]s/articles/1a-1 +Mon, 02 Jan 2006 15:04:05 GMTTeaser one +Seconda-2Tue, 03 Jan 2006 15:04:05 GMT +%[2]s

]]>
+
`, "http://"+r.Host, second) + }) + mux.HandleFunc("/b.atom", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/atom+xml") + _, _ = fmt.Fprintf(w, ` +Feed Burn:b2006-01-04T00:00:00Z +Atom entryurn:b:12006-01-04T00:00:00Z +Ada<h2>Atom</h2><p>body</p> +`) + }) + mux.HandleFunc("/articles/1", func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `Article one
+

Hello world

The full text of the first article is long enough for a readability extractor to keep it. +It carries several sentences, so the page is recognised as the main content and not as chrome.

+

A second paragraph makes the article clearly the densest block on the page.

`) + }) + s.Server = httptest.NewServer(mux) + t.Cleanup(s.Close) + return s +} + +// syncBoth runs the builtin connector and the plugin over the same config. +type harness struct { + t *testing.T + builtin *builtin.Connector + plugin *client.Client + cfg *types.DataSourceConfig +} + +func newHarness(t *testing.T) (*harness, *feedSite) { + t.Helper() + site := newFeedSite(t) + // The builtin connector fetches through the SSRF-safe client. + t.Setenv("SSRF_WHITELIST", "127.0.0.1") + utils.ResetSSRFWhitelistForTest() + t.Cleanup(utils.ResetSSRFWhitelistForTest) + + p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version}) + p.Connector("rss", &connector{}) + srv := httptest.NewServer(p.Handler()) + t.Cleanup(srv.Close) + return &harness{ + t: t, builtin: builtin.NewConnector(), plugin: client.New(srv.URL, nil, nil), + cfg: &types.DataSourceConfig{ + Type: types.ConnectorTypeRSS, + Settings: map[string]any{"feed_urls": site.URL + "/a.xml\n" + site.URL + "/b.atom"}, + }, + }, site +} + +func (h *harness) pluginFetch(mode string, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor) { + h.t.Helper() + instance := map[string]any{"settings": h.cfg.Settings, "credentials": map[string]any{}} + in := pluginapi.FetchInput{Mode: mode} + if cursor != nil { + in.Cursor = &pluginapi.Cursor{State: cursor.ConnectorCursor} + } + var items []types.FetchedItem + end, err := h.plugin.Stream(context.Background(), pluginapi.ConnectorFetchPath("rss"), + pluginapi.Envelope{Config: pluginapi.Config{Instance: instance}}, in, func(ev pluginapi.Event) error { + if ev.Type != pluginapi.EventItem { + return nil + } + var it pluginapi.FetchedItem + if err := json.Unmarshal(ev.Data, &it); err != nil { + return err + } + items = append(items, types.FetchedItem{ + ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType, + FileName: it.FileName, URL: it.URL, UpdatedAt: *it.UpdatedAt, Metadata: it.Metadata, + SourceResourceID: it.SourceResourceID, + }) + return nil + }) + if err != nil { + h.t.Fatalf("plugin fetch: %v", err) + } + var c pluginapi.Cursor + if err := json.Unmarshal(end, &c); err != nil { + h.t.Fatal(err) + } + return items, &types.SyncCursor{ConnectorCursor: c.State} +} + +// comparable renders items in a stable form; UpdatedAt of an undated item is +// "now" on both sides, so it is compared only when the feed dated the item. +func comparable(items []types.FetchedItem) []string { + var out []string + for _, it := range items { + meta, _ := json.Marshal(it.Metadata) + updated := it.UpdatedAt.UTC().Format(time.RFC3339) + if time.Since(it.UpdatedAt) < time.Hour { + updated = "now" + } + out = append(out, strings.Join([]string{ + it.ExternalID, it.Title, it.ContentType, it.FileName, it.URL, updated, it.SourceResourceID, + string(meta), string(it.Content), + }, " | ")) + } + sort.Strings(out) + return out +} + +func sameItems(t *testing.T, label string, want, got []types.FetchedItem) { + t.Helper() + w, g := comparable(want), comparable(got) + if strings.Join(w, "\n") != strings.Join(g, "\n") { + t.Fatalf("%s differ\nbuiltin:\n%s\n\nplugin:\n%s", label, strings.Join(w, "\n"), strings.Join(g, "\n")) + } +} + +func TestPluginMatchesBuiltinConnector(t *testing.T) { + ctx := context.Background() + h, site := newHarness(t) + + want, err := h.builtin.FetchAll(ctx, h.cfg, nil) + if err != nil { + t.Fatal(err) + } + got, pluginCursor := h.pluginFetch(pluginapi.FetchFull, nil) + if len(want) != 3 { + t.Fatalf("builtin fetched %d items, want 3", len(want)) + } + sameItems(t, "full sync", want, got) + if !strings.Contains( + string(got[0].Content)+string(got[1].Content)+string(got[2].Content), + "full text of the first article", + ) { + t.Fatal("the article page should replace the feed teaser") + } + + // An incremental run with nothing changed emits nothing on either side. + _, builtinCursor, err := h.builtin.FetchIncremental(ctx, h.cfg, nil) + if err != nil { + t.Fatal(err) + } + unchanged, _, err := h.builtin.FetchIncremental(ctx, h.cfg, builtinCursor) + if err != nil || len(unchanged) != 0 { + t.Fatalf("builtin incremental without changes = %d items, %v", len(unchanged), err) + } + if again, _ := h.pluginFetch(pluginapi.FetchIncremental, pluginCursor); len(again) != 0 { + t.Fatalf("plugin incremental without changes = %d items", len(again)) + } + + // One entry changes: both emit exactly that entry. + site.mu.Lock() + site.secondA = "Second post, edited." + site.mu.Unlock() + changedWant, _, err := h.builtin.FetchIncremental(ctx, h.cfg, builtinCursor) + if err != nil { + t.Fatal(err) + } + changedGot, _ := h.pluginFetch(pluginapi.FetchIncremental, pluginCursor) + if len(changedWant) != 1 { + t.Fatalf("builtin incremental after an edit = %d items", len(changedWant)) + } + sameItems(t, "incremental sync", changedWant, changedGot) +} + +func TestPluginValidatesLikeTheBuiltin(t *testing.T) { + h, site := newHarness(t) + ctx := context.Background() + call := func(settings map[string]any) error { + return h.plugin.Call(ctx, pluginapi.ConnectorValidatePath("rss"), pluginapi.Envelope{Config: pluginapi.Config{ + Instance: map[string]any{"settings": settings, "credentials": map[string]any{}}, + }}, nil, nil) + } + if err := call(h.cfg.Settings); err != nil { + t.Fatalf("valid feeds rejected: %v", err) + } + for name, settings := range map[string]map[string]any{ + "no feeds": {}, + "missing feed": {"feed_urls": site.URL + "/nope.xml"}, + } { + err := call(settings) + e, ok := pluginapi.AsError(err) + if !ok || e.Code != pluginapi.CodeInvalidConfig { + t.Errorf("%s: want invalid_config, got %v", name, err) + } + cfg := &types.DataSourceConfig{Settings: settings} + if h.builtin.Validate(ctx, cfg) == nil { + t.Errorf("%s: the builtin accepts what the plugin rejects", name) + } + } +} + +// The example must stay an external plugin: the SDK and ordinary libraries, +// nothing from WeKnora's internals. +func TestPluginImportsNoInternals(t *testing.T) { + files, _ := filepath.Glob("*.go") + fset := token.NewFileSet() + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + parsed, err := parser.ParseFile(fset, f, src, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, imp := range parsed.Imports { + if strings.Contains(imp.Path.Value, "github.com/Tencent/WeKnora/internal") { + t.Errorf("%s imports %s", f, imp.Path.Value) + } + } + } +} + +func TestPluginPassesConformance(t *testing.T) { + p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version}) + p.Connector("rss", &connector{}) + srv := httptest.NewServer(p.Handler()) + defer srv.Close() + rep := conformance.Run(context.Background(), conformance.Target{ + Client: client.New(srv.URL, nil, nil), + Raw: func(ctx context.Context, path string, body []byte) (*http.Response, error) { + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+path, bytes.NewReader(body)) + return http.DefaultClient.Do(req) + }, + }) + for _, r := range rep.Results { + if !r.Passed { + t.Errorf("%s: %s", r.Name, r.Detail) + } + } +} diff --git a/examples/plugins/rss/plugin.yaml b/examples/plugins/rss/plugin.yaml new file mode 100644 index 00000000..39991cf0 --- /dev/null +++ b/examples/plugins/rss/plugin.yaml @@ -0,0 +1,26 @@ +schemaVersion: 1 +id: weknora-examples.rss +version: 1.0.0 +apiVersion: weknora.plugin/v1 +name: { en-US: RSS (plugin), zh-CN: RSS 订阅(插件版) } +description: + en-US: Sync RSS, Atom and JSON feeds into a knowledge base. The builtin RSS connector, rebuilt as a code plugin. + zh-CN: 把 RSS、Atom、JSON Feed 同步到知识库。内置 RSS 连接器的插件实现。 +publisher: { id: weknora-examples, name: WeKnora examples } +homepage: https://github.com/Tencent/WeKnora/tree/main/examples/plugins/rss +license: MIT +icon: icon.svg +runtime: { type: host, kind: binary, entry: "bin/{os}-{arch}/rss" } +permissions: + # Feed items link to pages anywhere; private addresses stay unreachable. + egress: ["*"] +contributes: + connectors: + - id: rss + name: { en-US: RSS (plugin), zh-CN: RSS 订阅(插件版) } + description: + en-US: Articles from RSS, Atom and JSON feeds, as full text + zh-CN: 抓取 RSS、Atom、JSON Feed 的文章全文 + icon: icon.svg + capabilities: [incremental, streaming] + instanceSchema: schemas/instance.yaml diff --git a/examples/plugins/rss/schemas/instance.yaml b/examples/plugins/rss/schemas/instance.yaml new file mode 100644 index 00000000..e1675cec --- /dev/null +++ b/examples/plugins/rss/schemas/instance.yaml @@ -0,0 +1,22 @@ +type: object +required: [feed_urls] +properties: + feed_urls: + type: string + title: Feed URLs + description: One feed per line + x-widget: textarea + x-group: settings + x-placeholder: https://example.com/feed.xml + x-i18n: + title: { zh-CN: 订阅地址 } + description: { zh-CN: 每行一个 } + auth_headers: + type: string + title: Request headers + description: "Optional, one per line as Name: Value; sent to feed URLs only" + x-secret: true + x-widget: textarea + x-i18n: + title: { zh-CN: 请求头 } + description: { zh-CN: "可选,每行一个,格式为 名称: 值;只发给订阅地址" } diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts index fc3322c5..e3f86792 100644 --- a/frontend/src/api/datasource/index.ts +++ b/frontend/src/api/datasource/index.ts @@ -80,6 +80,13 @@ export interface ConnectorMeta { required_permissions?: string[] // Credential form: describes config.credentials for this connector. config_schema?: ConfigSchema + // Settings form (config.settings) for connectors whose settings UI is + // not built in, such as plugin connectors. + settings_schema?: ConfigSchema + // Localized name / description of plugin connectors, keyed by locale. + names?: Record + descriptions?: Record + plugin_id?: string } export interface Resource { @@ -126,8 +133,12 @@ export function validateConnection(id: string) { } // Validate credentials without persisting (during creation or credential replacement). -export function validateCredentials(type: string, credentials: Record) { - return post('/api/v1/datasource/validate-credentials', { type, credentials }) +export function validateCredentials( + type: string, + credentials: Record, + settings?: Record, +) { + return post('/api/v1/datasource/validate-credentials', { type, credentials, settings }) } // listResources lists selectable resources for a data source. Pass parentId to diff --git a/frontend/src/api/system/plugins.ts b/frontend/src/api/system/plugins.ts index 633ee560..fb29beac 100644 --- a/frontend/src/api/system/plugins.ts +++ b/frontend/src/api/system/plugins.ts @@ -31,7 +31,9 @@ export interface PluginVersion { /** This node's report on an installed plugin. */ export interface PluginNodeStatus { version: string - state: 'ready' | 'failed' + // degraded: loaded, but its process crashed or fails health checks and is + // being restarted. + state: 'ready' | 'failed' | 'degraded' error?: string updatedAt: string } diff --git a/frontend/src/api/web-search-provider.ts b/frontend/src/api/web-search-provider.ts index 73241067..7c4ae365 100644 --- a/frontend/src/api/web-search-provider.ts +++ b/frontend/src/api/web-search-provider.ts @@ -6,7 +6,8 @@ export interface WebSearchProviderEntity { id?: string tenant_id?: number name: string - provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply' + // A builtin provider ID, or a plugin provider's qualified ID (acme.search/brave). + provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply' | (string & {}) description?: string parameters: { // api_key is never returned by the server in this shape; it lives behind @@ -40,6 +41,10 @@ export interface WebSearchProviderTypeInfo { config_fields?: WebSearchProviderConfigField[] // The parameters form as a config schema; the settings drawer renders it. config_schema?: ConfigSchema + // Plugin provider types: localized names, an icon (data: URI), the plugin. + names?: Record + icon?: string + plugin_id?: string } export interface WebSearchProviderConfigField { diff --git a/frontend/src/i18n/localeKeyAudit.ts b/frontend/src/i18n/localeKeyAudit.ts index eefb2641..101f4ef7 100644 --- a/frontend/src/i18n/localeKeyAudit.ts +++ b/frontend/src/i18n/localeKeyAudit.ts @@ -116,6 +116,11 @@ const EXTRA_PREFIXES = [ // knowledgeEditor.wikiBrowser.gallery.attr.builtin_caption …, and // everything else falls back to the contract wording. 'knowledgeEditor.wikiBrowser.gallery.attr.', + // Connector names and descriptions are keyed by the connector type the + // backend registry lists (views/knowledge/settings/connectorLabels.ts); + // plugin connectors fall back to their own names. + 'datasource.connector.', + 'datasource.connectorDesc.', ] as const /** Keys that must survive pruning even when static analysis misses them. */ diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index c3247ec2..6d73064b 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -812,6 +812,8 @@ export default { channelGitLab: 'GitLab', channelIma: 'Tencent IMA', channelUpload: 'Upload', + channelRss: 'RSS', + channelExternal: 'External source', channelManual: 'Manual', channelUrl: 'Web', channelUnknown: 'Unknown', @@ -2732,6 +2734,10 @@ export default { } }, pluginAdmin: { + runtime: { + declarative: 'Declarative', + host: 'Local process' + }, title: 'Plugin management', description: 'Install and manage plugins for the whole platform. Every workspace sees an installed plugin, and each one enables it for itself; disabling a plugin here unloads it on every node.', installButton: 'Install plugin', @@ -2745,6 +2751,7 @@ export default { contributions: 'What it adds', permissions: 'Permissions and remote access', state: { + degraded: 'Restarting', running: 'Running', failed: 'Failed to load', disabled: 'Disabled', @@ -2767,6 +2774,7 @@ export default { reinstall: 'Reinstall' }, permission: { + anyHost: 'Any public host', remote: 'Remote service', egress: 'Network egress', hostApi: 'WeKnora API', diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts index a299aa85..191cc514 100644 --- a/frontend/src/i18n/locales/ja-JP.ts +++ b/frontend/src/i18n/locales/ja-JP.ts @@ -812,6 +812,8 @@ export default { channelGitLab: 'GitLab', channelIma: 'Tencent IMA', channelUpload: 'アップロード', + channelRss: 'RSS', + channelExternal: '外部データソース', channelManual: '手動作成', channelUrl: 'Web', channelUnknown: '不明', @@ -2732,6 +2734,10 @@ export default { } }, pluginAdmin: { + runtime: { + declarative: '宣言型', + host: 'ローカルプロセス' + }, title: 'プラグイン管理', description: 'プラットフォーム全体のプラグインをインストール・管理します。インストールしたプラグインはすべてのワークスペースに表示され、各ワークスペースで個別に有効化します。ここで無効にすると全ノードでアンロードされます。', installButton: 'プラグインをインストール', @@ -2745,6 +2751,7 @@ export default { contributions: '提供する機能', permissions: '権限と外部アクセス', state: { + degraded: '再起動中', running: '稼働中', failed: '読み込み失敗', disabled: '無効', @@ -2767,6 +2774,7 @@ export default { reinstall: '再インストール' }, permission: { + anyHost: '任意の公開ホスト', remote: 'リモートサービス', egress: 'ネットワーク送信', hostApi: 'WeKnora API', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 1b8e62b8..ce3083b4 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -5252,6 +5252,10 @@ export default { noCompatibleKbForAgent: '현재 에이전트의 도구와 범위 내 지식베이스의 기능이 일치하지 않아 참조할 수 있는 지식베이스가 없습니다.' }, pluginAdmin: { + runtime: { + declarative: '선언형', + host: '로컬 프로세스' + }, title: '플러그인 관리', description: '플랫폼 전체의 플러그인을 설치하고 관리합니다. 설치된 플러그인은 모든 워크스페이스에 보이며 각 워크스페이스가 직접 활성화합니다. 여기서 비활성화하면 모든 노드에서 언로드됩니다.', installButton: '플러그인 설치', @@ -5265,6 +5269,7 @@ export default { contributions: '제공 기능', permissions: '권한 및 외부 접근', state: { + degraded: '재시작 중', running: '실행 중', failed: '로드 실패', disabled: '비활성', @@ -5287,6 +5292,7 @@ export default { reinstall: '재설치' }, permission: { + anyHost: '모든 공개 호스트', remote: '원격 서비스', egress: '네트워크 송신', hostApi: 'WeKnora API', @@ -7315,6 +7321,8 @@ export default { channelGitLab: 'GitLab', channelIma: 'Tencent IMA', channelUpload: '업로드', + channelRss: 'RSS', + channelExternal: '외부 데이터 소스', channelManual: '수동', channelUrl: '웹', channelUnknown: '알 수 없음', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index c9a6e6a7..a8dffd3e 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -5252,6 +5252,10 @@ export default { noCompatibleKbForAgent: 'Инструменты текущего агента не соответствуют возможностям ни одной базы знаний в области видимости — нечего упомянуть.' }, pluginAdmin: { + runtime: { + declarative: 'Декларативный', + host: 'Локальный процесс' + }, title: 'Управление плагинами', description: 'Установка и управление плагинами всей платформы. Установленный плагин виден всем рабочим пространствам, и каждое включает его самостоятельно; отключение здесь выгружает плагин на всех узлах.', installButton: 'Установить плагин', @@ -5265,6 +5269,7 @@ export default { contributions: 'Что добавляет', permissions: 'Разрешения и внешний доступ', state: { + degraded: 'Перезапуск', running: 'Работает', failed: 'Ошибка загрузки', disabled: 'Отключён', @@ -5287,6 +5292,7 @@ export default { reinstall: 'Переустановка' }, permission: { + anyHost: 'Любой публичный хост', remote: 'Удалённый сервис', egress: 'Исходящая сеть', hostApi: 'API WeKnora', @@ -7315,6 +7321,8 @@ export default { channelGitLab: 'GitLab', channelIma: 'Tencent IMA', channelUpload: 'Загрузка', + channelRss: 'RSS', + channelExternal: 'Внешний источник', channelManual: 'Вручную', channelUrl: 'Веб', channelUnknown: 'Неизвестно', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index e2376815..6bf00c7b 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -5254,6 +5254,10 @@ export default { noCompatibleKbForAgent: '当前智能体的工具与作用域内知识库的能力不匹配,暂无可引用的知识库。' }, pluginAdmin: { + runtime: { + declarative: '声明式', + host: '本机进程' + }, title: '插件管理', description: '安装与管理本平台的插件。安装后所有空间都能看到,但每个空间需自行启用;停用会在所有节点卸载该插件。', installButton: '安装插件', @@ -5267,6 +5271,7 @@ export default { contributions: '提供的能力', permissions: '权限与外部访问', state: { + degraded: '重启中', running: '运行中', failed: '加载失败', disabled: '已停用', @@ -5289,6 +5294,7 @@ export default { reinstall: '重新安装' }, permission: { + anyHost: '任意公网地址', remote: '访问远程服务', egress: '网络出口', hostApi: '调用 WeKnora API', @@ -7317,6 +7323,8 @@ export default { channelGitLab: 'GitLab', channelIma: '腾讯 IMA', channelUpload: '上传', + channelRss: 'RSS 订阅', + channelExternal: '外部数据源', channelManual: '手动', channelUrl: '网页', channelUnknown: '未知', diff --git a/frontend/src/utils/localizedText.ts b/frontend/src/utils/localizedText.ts index edbf300a..6be9a9c5 100644 --- a/frontend/src/utils/localizedText.ts +++ b/frontend/src/utils/localizedText.ts @@ -9,3 +9,18 @@ export function localizedText(text: LocalizedText | undefined, locale: string): const match = Object.keys(text).sort().find((k) => k !== 'default' && k.split('-')[0] === lang && text[k]) return match ? text[match] : text.default } + +/** + * Picks a variant from a locale map without a default ({ 'zh-CN': ... }, as + * plugin connectors and provider types send names): exact, then same + * language, else undefined. + */ +export function pickLocale(variants: Record | undefined, locale: string): string | undefined { + if (!variants) return undefined + if (variants[locale]) return variants[locale] + const lang = locale.split('-')[0].toLowerCase() + const key = Object.keys(variants) + .sort() + .find((k) => k.split('-')[0].toLowerCase() === lang) + return key ? variants[key] : undefined +} diff --git a/frontend/src/views/knowledge/components/DocumentListView.vue b/frontend/src/views/knowledge/components/DocumentListView.vue index 6e94d74e..79f12b38 100644 --- a/frontend/src/views/knowledge/components/DocumentListView.vue +++ b/frontend/src/views/knowledge/components/DocumentListView.vue @@ -109,6 +109,9 @@ const getSourceInfo = (item: KnowledgeItem): { icon: string; label: string } => if (ch === 'dingtalk') return { icon: 'cloud-download', label: t('knowledgeBase.channelDingtalk') }; if (ch === 'slack') return { icon: 'cloud-download', label: t('knowledgeBase.channelSlack') }; if (ch === 'im') return { icon: 'cloud-download', label: t('knowledgeBase.channelIm') }; + if (ch === 'rss') return { icon: 'cloud-download', label: t('knowledgeBase.channelRss') }; + // Plugin connectors ingest under their qualified type ID (acme.feeds/feed). + if (ch && ch.includes('/')) return { icon: 'cloud-download', label: t('knowledgeBase.channelExternal') }; if (item.type === 'url') return { icon: 'link', label: t('knowledgeBase.channelUrl') }; if (item.type === 'manual') return { icon: 'edit', label: t('knowledgeBase.channelManual') }; return { icon: 'upload', label: t('knowledgeBase.channelUpload') }; diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts index d14d06fd..c935916f 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts @@ -80,7 +80,15 @@ async function fixture({ exports, require(name: string) { if (name === 'vue') return require('vue') - if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key }) } + if (name === 'vue-i18n') { + return { useI18n: () => ({ t: (key: string) => key, te: () => true, locale: { value: 'en-US' } }) } + } + if (name === './connectorLabels') { + return { + connectorName: (d: { type: string }) => `datasource.connector.${d.type}`, + connectorDescription: (d: { type: string }) => `datasource.connectorDesc.${d.type}`, + } + } if (name === 'tdesign-vue-next') return { MessagePlugin: { warning() {}, success() {}, error() {} } } if (name === '@/api/datasource') return api if (name === '@/components/schema-form/schema') return schemaModule diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue index fdf41aff..a7f038cc 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue @@ -29,6 +29,7 @@ import { import { useSchemaText } from '@/components/schema-form/useSchemaText' import SettingDrawer from '@/components/settings/SettingDrawer.vue' import DataSourceTypeIcon from './DataSourceTypeIcon.vue' +import { connectorDescription, connectorName } from './connectorLabels' import { getDatasourceIconUrl } from './datasourceIcons' const props = defineProps<{ @@ -38,7 +39,7 @@ const props = defineProps<{ const visible = defineModel('visible', { default: false }) const emit = defineEmits<{ saved: [] }>() -const { t } = useI18n() +const { t, te, locale } = useI18n() const isEdit = computed(() => !!props.dataSource) const step = ref(0) @@ -536,6 +537,11 @@ const currentDef = computed(() => connectorTypes.value.find(d => d.type === form const EMPTY_SCHEMA: ConfigSchema = { type: 'object', properties: {} } const credentialSchema = computed(() => currentDef.value?.config_schema ?? EMPTY_SCHEMA) +// Settings form of connectors without a built-in settings UI (plugins). +const settingsSchema = computed(() => currentDef.value?.settings_schema ?? null) +const settingsErrors = ref([]) +const labelI18n = computed(() => ({ t: (k: string) => t(k), te: (k: string) => te(k), locale: locale.value })) +const labelOf = (def: ConnectorMeta) => connectorName(def, labelI18n.value) // Field errors of the last credential check, shown under each field. const credentialErrors = ref([]) @@ -547,9 +553,12 @@ watch(() => form.value.config.credentials, () => { credentialErrors.value = [] } // fields and warns with the first one, as the editor always has. function checkCredentials(): boolean { credentialErrors.value = validateConfig(credentialSchema.value, form.value.config.credentials) - const first = credentialErrors.value[0] + settingsErrors.value = settingsSchema.value ? validateConfig(settingsSchema.value, form.value.config.settings) : [] + const inCredentials = credentialErrors.value.length > 0 + const first = credentialErrors.value[0] ?? settingsErrors.value[0] if (!first) return true - const field = schemaAt(credentialSchema.value, first.path) + const schema = inCredentials ? credentialSchema.value : (settingsSchema.value as ConfigSchema) + const field = schemaAt(schema, first.path) const label = field ? schemaText(field, 'title') : first.path MessagePlugin.warning( first.code === 'required' @@ -673,9 +682,13 @@ watch( function selectType(def: ConnectorMeta) { form.value.type = def.type - form.value.name = t(`datasource.connector.${def.type}`) + form.value.name = labelOf(def) // Schema defaults, e.g. Confluence starts on the Server edition. form.value.config.credentials = applyDefaults(def.config_schema ?? EMPTY_SCHEMA, {}) + if (def.settings_schema) { + form.value.config.settings = applyDefaults(def.settings_schema, form.value.config.settings ?? {}) + } + settingsErrors.value = [] credentialErrors.value = [] if (def.type === 'confluence') { form.value.config.settings = { ...form.value.config.settings, edition: 'server' } @@ -717,7 +730,7 @@ async function testConnection() { // validate-credentials is credentials-only; feed URLs live in settings. creds.feed_urls = form.value.config.settings.feed_urls } - await validateCredentials(form.value.type, creds) + await validateCredentials(form.value.type, creds, form.value.config.settings) } testResult.value = 'success' MessagePlugin.success(t('datasource.testSuccess')) @@ -1137,9 +1150,9 @@ const drawerConfirmText = computed(() => { @confirm="handleDrawerConfirm" @cancel="handleClose" > -