feat(plugin): code plugins — extension protocol, Go SDK, plugin host, web search and data source adapters (#3720)

* feat(pluginsdk): extension protocol v1, Go SDK and conformance suite
* feat(plugin): run code plugins in an embedded plugin host
* feat(plugin): remote adapters for web search and data sources
* fix(plugin): show plugin errors by their message
* feat(frontend): plugin connectors and web search providers in the UI
* docs(pluginsdk): how to write, package, test and install a plugin
* fix(plugin): gofmt two files and read openapi.yaml line-ending agnostic
* fix(plugin): mark a host plugin ready before Activate returns
main
lyingbug 2026-09-26 14:45:15 +08:00 committed by GitHub
parent ed63e35cc7
commit 4328ab26a9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
88 changed files with 6602 additions and 106 deletions

View File

@ -58,10 +58,10 @@ jobs:
modules=() modules=()
if git diff --name-only --diff-filter=ACMR "$diff_range" -- \ if git diff --name-only --diff-filter=ACMR "$diff_range" -- \
.golangci.yml .github/workflows/go-lint.yml | grep -q .; then .golangci.yml .github/workflows/go-lint.yml | grep -q .; then
modules=("." "cli" "client") modules=("." "cli" "client" "pluginsdk")
else else
if git diff --name-only --diff-filter=ACMR "$diff_range" -- \ if git diff --name-only --diff-filter=ACMR "$diff_range" -- \
'*.go' ':!cli/**' ':!client/**' go.mod go.sum | grep -q .; then '*.go' ':!cli/**' ':!client/**' ':!pluginsdk/**' go.mod go.sum | grep -q .; then
modules+=(".") modules+=(".")
fi fi
if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'cli/**' | grep -q .; then if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'cli/**' | grep -q .; then
@ -70,6 +70,9 @@ jobs:
if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'client/**' | grep -q .; then if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'client/**' | grep -q .; then
modules+=("client") modules+=("client")
fi fi
if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'pluginsdk/**' | grep -q .; then
modules+=("pluginsdk")
fi
fi fi
json="[" json="["
first=1 first=1

View File

@ -0,0 +1,44 @@
name: Plugin SDK
# Builds and tests the pluginsdk/ Go module (extension protocol, Go SDK,
# client and conformance suite) on every push / PR that touches it.
on:
push:
branches: [main]
paths:
- 'pluginsdk/**'
- '.github/workflows/plugin-sdk.yml'
pull_request:
paths:
- 'pluginsdk/**'
- '.github/workflows/plugin-sdk.yml'
defaults:
run:
working-directory: pluginsdk
shell: bash
permissions:
contents: read
jobs:
test:
name: test (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v6
- uses: actions/setup-go@v6
with:
go-version: '1.26'
cache-dependency-path: pluginsdk/go.mod
- name: go build
run: go build ./...
- name: go test
run: go test -race ./...
- name: go vet
run: go vet ./...

View File

@ -38,10 +38,12 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
# Install migrate tool # Install migrate tool
RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
# Copy go mod files. go.mod replace-points anydoc at ./third_party/anydoc-go, # Copy go mod files. go.mod replace-points anydoc at ./third_party/anydoc-go
# so that module's go.mod must exist before `go mod download`. # and the plugin SDK at ./pluginsdk, so their go.mod files must exist before
# `go mod download`.
COPY go.mod go.sum ./ COPY go.mod go.sum ./
COPY third_party/anydoc-go/go.mod third_party/anydoc-go/go.mod COPY third_party/anydoc-go/go.mod third_party/anydoc-go/go.mod
COPY pluginsdk/go.mod pluginsdk/go.mod
RUN --mount=type=cache,target=/go/pkg/mod go mod download RUN --mount=type=cache,target=/go/pkg/mod go mod download
COPY cmd/download cmd/download COPY cmd/download cmd/download
RUN go run cmd/download/duckdb/duckdb.go RUN go run cmd/download/duckdb/duckdb.go

View File

@ -0,0 +1,282 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"maps"
"net/http"
"net/url"
"strings"
"time"
"unicode/utf8"
readability "codeberg.org/readeck/go-readability/v2"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/mmcdole/gofeed"
)
const (
requestTimeout = 20 * time.Second
maxFeedSize = 10 << 20
maxArticleSize = 5 << 20
defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
maxFileNameBytes = 200
)
// config is the data source configuration: feed URLs in settings, optional
// auth headers (which may carry secrets) in credentials.
type config struct {
feedURLs []string
authHeaders map[string]string
}
func parseConfig(settings, credentials map[string]any) (*config, error) {
raw, _ := settings["feed_urls"].(string)
if strings.TrimSpace(raw) == "" {
// Older rows kept the URLs with the credentials.
raw, _ = credentials["feed_urls"].(string)
}
cfg := &config{feedURLs: splitURLs(raw)}
if len(cfg.feedURLs) == 0 {
return nil, errMissingFeeds
}
headers, _ := credentials["auth_headers"].(string)
cfg.authHeaders = parseHeaders(headers)
return cfg, nil
}
// splitURLs splits on newlines and commas, trims, dedupes (order kept).
func splitURLs(raw string) []string {
parts := strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' })
seen := map[string]bool{}
var out []string
for _, u := range parts {
u = strings.TrimSpace(u)
if u == "" || seen[u] {
continue
}
seen[u] = true
out = append(out, u)
}
return out
}
func parseHeaders(raw string) map[string]string {
headers := map[string]string{}
for _, line := range strings.Split(raw, "\n") {
name, value, ok := strings.Cut(strings.TrimSpace(line), ":")
name = strings.TrimSpace(name)
if !ok || name == "" {
continue
}
headers[name] = strings.TrimSpace(value)
}
if len(headers) == 0 {
return nil
}
return headers
}
// fetcher downloads feeds and article pages. Outbound traffic goes through
// the plugin host's egress proxy (HTTPS_PROXY), which refuses private
// addresses, so the plugin needs no SSRF logic of its own.
type fetcher struct {
http *http.Client
headers map[string]string
}
func newFetcher(headers map[string]string) *fetcher {
return &fetcher{http: &http.Client{Timeout: requestTimeout}, headers: headers}
}
// fetch attaches auth headers only to feed fetches; article pages on
// third-party sites must not receive feed credentials.
func (f *fetcher) fetch(ctx context.Context, rawURL string, maxSize int64, withAuth bool) ([]byte, error) {
u, err := url.Parse(rawURL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") {
return nil, fmt.Errorf("invalid URL %q", rawURL)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, err
}
if withAuth {
for k, v := range f.headers {
req.Header.Set(k, v)
}
}
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", defaultUserAgent)
}
if req.Header.Get("Accept") == "" {
req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, "+
"application/json, text/html;q=0.9, */*;q=0.8")
}
resp, err := f.http.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch failed: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1))
if err != nil {
return nil, fmt.Errorf("read body failed: %w", err)
}
if int64(len(body)) > maxSize {
return nil, fmt.Errorf("response exceeds maximum size (%d bytes)", maxSize)
}
return body, nil
}
func (f *fetcher) feed(ctx context.Context, feedURL string) (*gofeed.Feed, error) {
data, err := f.fetch(ctx, feedURL, maxFeedSize, true)
if err != nil {
return nil, err
}
feed, err := gofeed.NewParser().Parse(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("parse feed: %w", err)
}
return feed, nil
}
// article returns the readability-cleaned main content of a page as HTML.
func (f *fetcher) article(ctx context.Context, articleURL string) (string, string, error) {
body, err := f.fetch(ctx, articleURL, maxArticleSize, false)
if err != nil {
return "", "", err
}
pageURL, _ := url.Parse(articleURL)
art, err := readability.FromReader(bytes.NewReader(body), pageURL)
if err != nil {
return "", "", fmt.Errorf("readability parse: %w", err)
}
if art.Node == nil {
return "", "", fmt.Errorf("no readable content extracted")
}
var buf bytes.Buffer
if err := art.RenderHTML(&buf); err != nil {
return "", "", fmt.Errorf("render article html: %w", err)
}
return buf.String(), art.Title(), nil
}
// cursorState is the incremental sync state, the same shape the builtin
// connector keeps: feed URL → item ID → content fingerprint, and feed URL →
// item ID → feed-only signal that lets unchanged entries skip article fetches.
type cursorState struct {
FeedItems map[string]map[string]string
FeedSignals map[string]map[string]string
}
func newCursorState() *cursorState {
return &cursorState{FeedItems: map[string]map[string]string{}, FeedSignals: map[string]map[string]string{}}
}
func cursorFromState(state map[string]any) *cursorState {
c := newCursorState()
read := func(key string, into map[string]map[string]string) {
feeds, _ := state[key].(map[string]any)
for feedURL, items := range feeds {
m, _ := items.(map[string]any)
into[feedURL] = map[string]string{}
for id, v := range m {
if s, ok := v.(string); ok {
into[feedURL][id] = s
}
}
}
}
read("feed_items", c.FeedItems)
read("feed_signals", c.FeedSignals)
return c
}
func (c *cursorState) toState() map[string]any {
return map[string]any{"feed_items": c.FeedItems, "feed_signals": c.FeedSignals}
}
func (c *cursorState) copyFeed(prev *cursorState, feedURL string) {
if prev == nil {
return
}
if src := prev.FeedItems[feedURL]; len(src) > 0 {
c.FeedItems[feedURL] = maps.Clone(src)
}
if src := prev.FeedSignals[feedURL]; len(src) > 0 {
c.FeedSignals[feedURL] = maps.Clone(src)
}
}
func contentFingerprint(markdown string) string {
sum := sha256.Sum256([]byte(markdown))
return "h:" + hex.EncodeToString(sum[:])[:16]
}
func feedSignal(item *gofeed.Item, feedContent string) string {
var b strings.Builder
b.WriteString(item.GUID + "\n" + item.Link + "\n" + item.Title + "\n")
if item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero() {
b.WriteString(item.UpdatedParsed.UTC().Format(time.RFC3339))
}
b.WriteByte('\n')
if item.PublishedParsed != nil && !item.PublishedParsed.IsZero() {
b.WriteString(item.PublishedParsed.UTC().Format(time.RFC3339))
}
b.WriteByte('\n')
b.WriteString(feedContent)
sum := sha256.Sum256([]byte(b.String()))
return "s:" + hex.EncodeToString(sum[:])[:16]
}
func firstNonEmpty(values ...string) string {
for _, v := range values {
if strings.TrimSpace(v) != "" {
return v
}
}
return ""
}
func htmlToMarkdown(html string) string {
if strings.TrimSpace(html) == "" {
return ""
}
md, err := htmltomd.ConvertString(html)
if err != nil || strings.TrimSpace(md) == "" {
return strings.TrimSpace(html)
}
return strings.TrimSpace(md)
}
var fileNameReplacer = strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
)
// fileName makes a feed title safe to use as a file name, the way WeKnora's
// builtin connectors do: hostile punctuation replaced, at most 200 bytes
// without splitting a rune.
func fileName(title string) string {
title = strings.TrimSpace(strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(title))
if title == "" {
return "untitled.md"
}
name := fileNameReplacer.Replace(title)
if len(name) > maxFileNameBytes {
name = name[:maxFileNameBytes]
for len(name) > 0 {
r, size := utf8.DecodeLastRuneInString(name)
if r != utf8.RuneError || size != 1 {
break
}
name = name[:len(name)-1]
}
}
return name + ".md"
}

View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><rect width="24" height="24" rx="5" fill="#f26522"/><circle cx="6.5" cy="17.5" r="2" fill="#fff"/><path d="M4.5 10.5a9 9 0 0 1 9 9h-2.6a6.4 6.4 0 0 0-6.4-6.4z" fill="#fff"/><path d="M4.5 4.5a15 15 0 0 1 15 15h-2.6A12.4 12.4 0 0 0 4.5 7.1z" fill="#fff"/></svg>

After

Width:  |  Height:  |  Size: 320 B

View File

@ -0,0 +1,229 @@
// Command rss is WeKnora's RSS/Atom/JSON feed connector written as an
// external plugin with the Go SDK. It behaves like the builtin rss connector
// (a test syncs the same feeds through both) and shows how a code plugin is
// built: it imports only the SDK and ordinary libraries, never WeKnora's
// internals.
//
// Build a package with ./package.sh; install it under System administration
// → Plugin management.
package main
import (
"context"
"errors"
"fmt"
"log"
"strings"
"time"
"github.com/mmcdole/gofeed"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Version must match plugin.yaml.
const Version = "1.0.0"
var errMissingFeeds = pluginapi.InvalidConfig(
"feed_urls is required",
map[string]string{"settings.feed_urls": "required"},
)
func main() {
p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version})
p.Connector("rss", &connector{logf: p.Logger().Warn})
if err := p.Serve(); err != nil {
log.Fatal(err)
}
}
type connector struct {
logf func(msg string, args ...any)
}
func (c *connector) warn(msg string, args ...any) {
if c.logf != nil {
c.logf(msg, args...)
}
}
// Validate fetches and parses every configured feed.
func (c *connector) Validate(ctx context.Context, _ *pluginsdk.Call, in pluginsdk.ConnectorConfig) error {
cfg, err := parseConfig(in.Settings, in.Credentials)
if err != nil {
return err
}
f := newFetcher(cfg.authHeaders)
for _, u := range cfg.feedURLs {
if _, err := f.feed(ctx, u); err != nil {
return pluginapi.InvalidConfig(fmt.Sprintf("feed %s: %v", u, err),
map[string]string{"settings.feed_urls": err.Error()})
}
}
return nil
}
// ListResources lists one resource per feed. A feed that fails still
// appears, named by its URL, so the user can deselect it.
func (c *connector) ListResources(
ctx context.Context, _ *pluginsdk.Call, in pluginsdk.ConnectorConfig, parentID string,
) ([]pluginapi.Resource, error) {
if parentID != "" {
return nil, nil
}
cfg, err := parseConfig(in.Settings, in.Credentials)
if err != nil {
return nil, err
}
f := newFetcher(cfg.authHeaders)
out := make([]pluginapi.Resource, 0, len(cfg.feedURLs))
for _, u := range cfg.feedURLs {
res := pluginapi.Resource{ExternalID: u, Type: "feed", Name: u, URL: u}
feed, err := f.feed(ctx, u)
if err != nil {
res.Description = "fetch failed: " + err.Error()
out = append(out, res)
continue
}
if t := strings.TrimSpace(feed.Title); t != "" {
res.Name = t
}
res.Description = strings.TrimSpace(feed.Description)
if feed.Link != "" {
res.URL = feed.Link
}
res.ModifiedAt = feed.UpdatedParsed
res.Metadata = map[string]any{"item_count": len(feed.Items)}
out = append(out, res)
}
return out, nil
}
// Fetch syncs the selected feeds (all configured ones when none are
// selected). Incremental runs skip entries whose feed signal and content are
// unchanged; deletions are never emitted, since feeds drop old items as a
// matter of course.
func (c *connector) Fetch(
ctx context.Context,
_ *pluginsdk.Call,
in pluginsdk.ConnectorConfig,
req pluginapi.FetchInput,
s *pluginsdk.Stream,
) (*pluginapi.Cursor, error) {
cfg, err := parseConfig(in.Settings, in.Credentials)
if err != nil {
return nil, err
}
feedURLs := req.ResourceIDs
if len(feedURLs) == 0 {
feedURLs = cfg.feedURLs
}
var prev *cursorState
incremental := req.Mode == pluginapi.FetchIncremental && req.Cursor != nil
if incremental {
prev = cursorFromState(req.Cursor.State)
}
f := newFetcher(cfg.authHeaders)
next := newCursorState()
now := time.Now().UTC()
var failures []string
for _, feedURL := range feedURLs {
feed, err := f.feed(ctx, feedURL)
if err != nil {
c.warn("feed failed", "url", feedURL, "err", err)
failures = append(failures, fmt.Sprintf("%s: %v", feedURL, err))
next.copyFeed(prev, feedURL)
continue
}
next.FeedItems[feedURL] = map[string]string{}
next.FeedSignals[feedURL] = map[string]string{}
var prevItems, prevSignals map[string]string
if prev != nil {
prevItems, prevSignals = prev.FeedItems[feedURL], prev.FeedSignals[feedURL]
}
for _, item := range feed.Items {
if item == nil {
continue
}
itemID := firstNonEmpty(item.GUID, item.Link, item.Title)
if itemID == "" {
continue
}
feedContent := firstNonEmpty(item.Content, item.Description)
sig := feedSignal(item, feedContent)
if incremental && prevItems[itemID] != "" && prevSignals[itemID] == sig {
next.FeedItems[feedURL][itemID] = prevItems[itemID]
next.FeedSignals[feedURL][itemID] = sig
continue
}
fetched, fp := c.resolve(ctx, f, feed, item, feedURL, itemID, feedContent)
next.FeedItems[feedURL][itemID] = fp
next.FeedSignals[feedURL][itemID] = sig
if incremental && prevItems[itemID] == fp {
continue
}
if err := s.Item(fetched); err != nil {
return nil, err
}
}
// Each finished feed is a safe place to resume from.
if err := s.Checkpoint(pluginapi.Cursor{LastSyncTime: &now, State: next.toState()}); err != nil {
return nil, err
}
}
if len(failures) > 0 && len(failures) == len(feedURLs) {
return nil, pluginapi.Errorf(pluginapi.CodeUnavailable, "all feeds failed: %s", strings.Join(failures, "; "))
}
if len(failures) > 0 {
_ = s.Log("warn", "some feeds failed: "+strings.Join(failures, "; "))
}
return &pluginapi.Cursor{LastSyncTime: &now, State: next.toState()}, nil
}
// resolve builds one item: the article's full text when it can be fetched,
// the feed's own content otherwise, as Markdown.
func (c *connector) resolve(
ctx context.Context, f *fetcher, feed *gofeed.Feed, item *gofeed.Item, feedURL, itemID, feedContent string,
) (pluginapi.FetchedItem, string) {
title := firstNonEmpty(item.Title, "untitled")
contentHTML := feedContent
if strings.TrimSpace(item.Link) != "" {
html, articleTitle, err := f.article(ctx, item.Link)
if err == nil {
contentHTML = html
if item.Title == "" && articleTitle != "" {
title = articleTitle
}
} else if !errors.Is(err, context.Canceled) {
c.warn("full-text fetch failed, using feed content", "url", item.Link, "err", err)
}
}
content := htmlToMarkdown(contentHTML)
updated := time.Now().UTC()
switch {
case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero():
updated = *item.UpdatedParsed
case item.PublishedParsed != nil && !item.PublishedParsed.IsZero():
updated = *item.PublishedParsed
}
author := ""
if item.Author != nil {
author = item.Author.Name
}
return pluginapi.FetchedItem{
ExternalID: feedURL + ":" + itemID,
Title: title,
Content: []byte(content),
ContentType: "text/markdown",
FileName: fileName(title),
URL: item.Link,
UpdatedAt: &updated,
SourceResourceID: feedURL,
Metadata: map[string]string{
"channel": "rss", "feed_url": feedURL, "feed_title": feed.Title,
"guid": item.GUID, "link": item.Link, "author": author,
},
}, contentFingerprint(content)
}

View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
# Builds the RSS example plugin for the platforms WeKnora runs on and zips it
# into a .wkp package: plugin.yaml, schemas, icon and bin/<os>-<arch>/rss.
set -euo pipefail
cd "$(dirname "$0")"
version=$(sed -n 's/^version: //p' plugin.yaml)
out=${1:-"weknora-examples-rss-${version}.wkp"}
stage=$(mktemp -d)
trap 'rm -rf "$stage"' EXIT
cp plugin.yaml icon.svg "$stage"/
cp -r schemas "$stage"/
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64; do
os=${target%/*}
arch=${target#*/}
bin="$stage/bin/${os}-${arch}/rss"
[ "$os" = windows ] && bin="$bin.exe"
CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -trimpath -ldflags="-s -w" -o "$bin" .
done
rm -f "$out"
(cd "$stage" && zip -qr - .) > "$out"
echo "wrote $out"

View File

@ -0,0 +1,279 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"go/parser"
"go/token"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"testing"
"time"
builtin "github.com/Tencent/WeKnora/internal/datasource/connector/rss"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/conformance"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// feedSite serves two feeds and the article pages they link to. Changing an
// entry's text simulates a feed update between syncs.
type feedSite struct {
*httptest.Server
mu sync.Mutex
secondA string
}
func newFeedSite(t *testing.T) *feedSite {
s := &feedSite{secondA: "Second post, first version."}
mux := http.NewServeMux()
mux.HandleFunc("/a.xml", func(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
second := s.secondA
s.mu.Unlock()
w.Header().Set("Content-Type", "application/rss+xml")
_, _ = fmt.Fprintf(w, `<?xml version="1.0"?><rss version="2.0"><channel><title>Feed A</title><link>%[1]s</link>
<description>First feed</description>
<item><title>Hello: world</title><link>%[1]s/articles/1</link><guid>a-1</guid>
<pubDate>Mon, 02 Jan 2006 15:04:05 GMT</pubDate><description>Teaser one</description></item>
<item><title>Second</title><guid>a-2</guid><pubDate>Tue, 03 Jan 2006 15:04:05 GMT</pubDate>
<description><![CDATA[<p>%[2]s</p>]]></description></item>
</channel></rss>`, "http://"+r.Host, second)
})
mux.HandleFunc("/b.atom", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/atom+xml")
_, _ = fmt.Fprintf(w, `<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom">
<title>Feed B</title><id>urn:b</id><updated>2006-01-04T00:00:00Z</updated>
<entry><title>Atom entry</title><id>urn:b:1</id><updated>2006-01-04T00:00:00Z</updated>
<author><name>Ada</name></author><content type="html">&lt;h2&gt;Atom&lt;/h2&gt;&lt;p&gt;body&lt;/p&gt;</content></entry>
</feed>`)
})
mux.HandleFunc("/articles/1", func(w http.ResponseWriter, _ *http.Request) {
_, _ = fmt.Fprint(w, `<html><head><title>Article one</title></head><body><nav>menu</nav><article>
<h1>Hello world</h1><p>The full text of the first article is long enough for a readability extractor to keep it.
It carries several sentences, so the page is recognised as the main content and not as chrome.</p>
<p>A second paragraph makes the article clearly the densest block on the page.</p></article></body></html>`)
})
s.Server = httptest.NewServer(mux)
t.Cleanup(s.Close)
return s
}
// syncBoth runs the builtin connector and the plugin over the same config.
type harness struct {
t *testing.T
builtin *builtin.Connector
plugin *client.Client
cfg *types.DataSourceConfig
}
func newHarness(t *testing.T) (*harness, *feedSite) {
t.Helper()
site := newFeedSite(t)
// The builtin connector fetches through the SSRF-safe client.
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
utils.ResetSSRFWhitelistForTest()
t.Cleanup(utils.ResetSSRFWhitelistForTest)
p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version})
p.Connector("rss", &connector{})
srv := httptest.NewServer(p.Handler())
t.Cleanup(srv.Close)
return &harness{
t: t, builtin: builtin.NewConnector(), plugin: client.New(srv.URL, nil, nil),
cfg: &types.DataSourceConfig{
Type: types.ConnectorTypeRSS,
Settings: map[string]any{"feed_urls": site.URL + "/a.xml\n" + site.URL + "/b.atom"},
},
}, site
}
func (h *harness) pluginFetch(mode string, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor) {
h.t.Helper()
instance := map[string]any{"settings": h.cfg.Settings, "credentials": map[string]any{}}
in := pluginapi.FetchInput{Mode: mode}
if cursor != nil {
in.Cursor = &pluginapi.Cursor{State: cursor.ConnectorCursor}
}
var items []types.FetchedItem
end, err := h.plugin.Stream(context.Background(), pluginapi.ConnectorFetchPath("rss"),
pluginapi.Envelope{Config: pluginapi.Config{Instance: instance}}, in, func(ev pluginapi.Event) error {
if ev.Type != pluginapi.EventItem {
return nil
}
var it pluginapi.FetchedItem
if err := json.Unmarshal(ev.Data, &it); err != nil {
return err
}
items = append(items, types.FetchedItem{
ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType,
FileName: it.FileName, URL: it.URL, UpdatedAt: *it.UpdatedAt, Metadata: it.Metadata,
SourceResourceID: it.SourceResourceID,
})
return nil
})
if err != nil {
h.t.Fatalf("plugin fetch: %v", err)
}
var c pluginapi.Cursor
if err := json.Unmarshal(end, &c); err != nil {
h.t.Fatal(err)
}
return items, &types.SyncCursor{ConnectorCursor: c.State}
}
// comparable renders items in a stable form; UpdatedAt of an undated item is
// "now" on both sides, so it is compared only when the feed dated the item.
func comparable(items []types.FetchedItem) []string {
var out []string
for _, it := range items {
meta, _ := json.Marshal(it.Metadata)
updated := it.UpdatedAt.UTC().Format(time.RFC3339)
if time.Since(it.UpdatedAt) < time.Hour {
updated = "now"
}
out = append(out, strings.Join([]string{
it.ExternalID, it.Title, it.ContentType, it.FileName, it.URL, updated, it.SourceResourceID,
string(meta), string(it.Content),
}, " | "))
}
sort.Strings(out)
return out
}
func sameItems(t *testing.T, label string, want, got []types.FetchedItem) {
t.Helper()
w, g := comparable(want), comparable(got)
if strings.Join(w, "\n") != strings.Join(g, "\n") {
t.Fatalf("%s differ\nbuiltin:\n%s\n\nplugin:\n%s", label, strings.Join(w, "\n"), strings.Join(g, "\n"))
}
}
func TestPluginMatchesBuiltinConnector(t *testing.T) {
ctx := context.Background()
h, site := newHarness(t)
want, err := h.builtin.FetchAll(ctx, h.cfg, nil)
if err != nil {
t.Fatal(err)
}
got, pluginCursor := h.pluginFetch(pluginapi.FetchFull, nil)
if len(want) != 3 {
t.Fatalf("builtin fetched %d items, want 3", len(want))
}
sameItems(t, "full sync", want, got)
if !strings.Contains(
string(got[0].Content)+string(got[1].Content)+string(got[2].Content),
"full text of the first article",
) {
t.Fatal("the article page should replace the feed teaser")
}
// An incremental run with nothing changed emits nothing on either side.
_, builtinCursor, err := h.builtin.FetchIncremental(ctx, h.cfg, nil)
if err != nil {
t.Fatal(err)
}
unchanged, _, err := h.builtin.FetchIncremental(ctx, h.cfg, builtinCursor)
if err != nil || len(unchanged) != 0 {
t.Fatalf("builtin incremental without changes = %d items, %v", len(unchanged), err)
}
if again, _ := h.pluginFetch(pluginapi.FetchIncremental, pluginCursor); len(again) != 0 {
t.Fatalf("plugin incremental without changes = %d items", len(again))
}
// One entry changes: both emit exactly that entry.
site.mu.Lock()
site.secondA = "Second post, edited."
site.mu.Unlock()
changedWant, _, err := h.builtin.FetchIncremental(ctx, h.cfg, builtinCursor)
if err != nil {
t.Fatal(err)
}
changedGot, _ := h.pluginFetch(pluginapi.FetchIncremental, pluginCursor)
if len(changedWant) != 1 {
t.Fatalf("builtin incremental after an edit = %d items", len(changedWant))
}
sameItems(t, "incremental sync", changedWant, changedGot)
}
func TestPluginValidatesLikeTheBuiltin(t *testing.T) {
h, site := newHarness(t)
ctx := context.Background()
call := func(settings map[string]any) error {
return h.plugin.Call(ctx, pluginapi.ConnectorValidatePath("rss"), pluginapi.Envelope{Config: pluginapi.Config{
Instance: map[string]any{"settings": settings, "credentials": map[string]any{}},
}}, nil, nil)
}
if err := call(h.cfg.Settings); err != nil {
t.Fatalf("valid feeds rejected: %v", err)
}
for name, settings := range map[string]map[string]any{
"no feeds": {},
"missing feed": {"feed_urls": site.URL + "/nope.xml"},
} {
err := call(settings)
e, ok := pluginapi.AsError(err)
if !ok || e.Code != pluginapi.CodeInvalidConfig {
t.Errorf("%s: want invalid_config, got %v", name, err)
}
cfg := &types.DataSourceConfig{Settings: settings}
if h.builtin.Validate(ctx, cfg) == nil {
t.Errorf("%s: the builtin accepts what the plugin rejects", name)
}
}
}
// The example must stay an external plugin: the SDK and ordinary libraries,
// nothing from WeKnora's internals.
func TestPluginImportsNoInternals(t *testing.T) {
files, _ := filepath.Glob("*.go")
fset := token.NewFileSet()
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
parsed, err := parser.ParseFile(fset, f, src, parser.ImportsOnly)
if err != nil {
t.Fatal(err)
}
for _, imp := range parsed.Imports {
if strings.Contains(imp.Path.Value, "github.com/Tencent/WeKnora/internal") {
t.Errorf("%s imports %s", f, imp.Path.Value)
}
}
}
}
func TestPluginPassesConformance(t *testing.T) {
p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version})
p.Connector("rss", &connector{})
srv := httptest.NewServer(p.Handler())
defer srv.Close()
rep := conformance.Run(context.Background(), conformance.Target{
Client: client.New(srv.URL, nil, nil),
Raw: func(ctx context.Context, path string, body []byte) (*http.Response, error) {
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+path, bytes.NewReader(body))
return http.DefaultClient.Do(req)
},
})
for _, r := range rep.Results {
if !r.Passed {
t.Errorf("%s: %s", r.Name, r.Detail)
}
}
}

View File

@ -0,0 +1,26 @@
schemaVersion: 1
id: weknora-examples.rss
version: 1.0.0
apiVersion: weknora.plugin/v1
name: { en-US: RSS (plugin), zh-CN: RSS 订阅(插件版) }
description:
en-US: Sync RSS, Atom and JSON feeds into a knowledge base. The builtin RSS connector, rebuilt as a code plugin.
zh-CN: 把 RSS、Atom、JSON Feed 同步到知识库。内置 RSS 连接器的插件实现。
publisher: { id: weknora-examples, name: WeKnora examples }
homepage: https://github.com/Tencent/WeKnora/tree/main/examples/plugins/rss
license: MIT
icon: icon.svg
runtime: { type: host, kind: binary, entry: "bin/{os}-{arch}/rss" }
permissions:
# Feed items link to pages anywhere; private addresses stay unreachable.
egress: ["*"]
contributes:
connectors:
- id: rss
name: { en-US: RSS (plugin), zh-CN: RSS 订阅(插件版) }
description:
en-US: Articles from RSS, Atom and JSON feeds, as full text
zh-CN: 抓取 RSS、Atom、JSON Feed 的文章全文
icon: icon.svg
capabilities: [incremental, streaming]
instanceSchema: schemas/instance.yaml

View File

@ -0,0 +1,22 @@
type: object
required: [feed_urls]
properties:
feed_urls:
type: string
title: Feed URLs
description: One feed per line
x-widget: textarea
x-group: settings
x-placeholder: https://example.com/feed.xml
x-i18n:
title: { zh-CN: 订阅地址 }
description: { zh-CN: 每行一个 }
auth_headers:
type: string
title: Request headers
description: "Optional, one per line as Name: Value; sent to feed URLs only"
x-secret: true
x-widget: textarea
x-i18n:
title: { zh-CN: 请求头 }
description: { zh-CN: "可选,每行一个,格式为 名称: 值;只发给订阅地址" }

View File

@ -80,6 +80,13 @@ export interface ConnectorMeta {
required_permissions?: string[] required_permissions?: string[]
// Credential form: describes config.credentials for this connector. // Credential form: describes config.credentials for this connector.
config_schema?: ConfigSchema config_schema?: ConfigSchema
// Settings form (config.settings) for connectors whose settings UI is
// not built in, such as plugin connectors.
settings_schema?: ConfigSchema
// Localized name / description of plugin connectors, keyed by locale.
names?: Record<string, string>
descriptions?: Record<string, string>
plugin_id?: string
} }
export interface Resource { export interface Resource {
@ -126,8 +133,12 @@ export function validateConnection(id: string) {
} }
// Validate credentials without persisting (during creation or credential replacement). // Validate credentials without persisting (during creation or credential replacement).
export function validateCredentials(type: string, credentials: Record<string, any>) { export function validateCredentials(
return post('/api/v1/datasource/validate-credentials', { type, credentials }) type: string,
credentials: Record<string, any>,
settings?: Record<string, any>,
) {
return post('/api/v1/datasource/validate-credentials', { type, credentials, settings })
} }
// listResources lists selectable resources for a data source. Pass parentId to // listResources lists selectable resources for a data source. Pass parentId to

View File

@ -31,7 +31,9 @@ export interface PluginVersion {
/** This node's report on an installed plugin. */ /** This node's report on an installed plugin. */
export interface PluginNodeStatus { export interface PluginNodeStatus {
version: string version: string
state: 'ready' | 'failed' // degraded: loaded, but its process crashed or fails health checks and is
// being restarted.
state: 'ready' | 'failed' | 'degraded'
error?: string error?: string
updatedAt: string updatedAt: string
} }

View File

@ -6,7 +6,8 @@ export interface WebSearchProviderEntity {
id?: string id?: string
tenant_id?: number tenant_id?: number
name: string name: string
provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply' // A builtin provider ID, or a plugin provider's qualified ID (acme.search/brave).
provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply' | (string & {})
description?: string description?: string
parameters: { parameters: {
// api_key is never returned by the server in this shape; it lives behind // api_key is never returned by the server in this shape; it lives behind
@ -40,6 +41,10 @@ export interface WebSearchProviderTypeInfo {
config_fields?: WebSearchProviderConfigField[] config_fields?: WebSearchProviderConfigField[]
// The parameters form as a config schema; the settings drawer renders it. // The parameters form as a config schema; the settings drawer renders it.
config_schema?: ConfigSchema config_schema?: ConfigSchema
// Plugin provider types: localized names, an icon (data: URI), the plugin.
names?: Record<string, string>
icon?: string
plugin_id?: string
} }
export interface WebSearchProviderConfigField { export interface WebSearchProviderConfigField {

View File

@ -116,6 +116,11 @@ const EXTRA_PREFIXES = [
// knowledgeEditor.wikiBrowser.gallery.attr.builtin_caption …, and // knowledgeEditor.wikiBrowser.gallery.attr.builtin_caption …, and
// everything else falls back to the contract wording. // everything else falls back to the contract wording.
'knowledgeEditor.wikiBrowser.gallery.attr.', 'knowledgeEditor.wikiBrowser.gallery.attr.',
// Connector names and descriptions are keyed by the connector type the
// backend registry lists (views/knowledge/settings/connectorLabels.ts);
// plugin connectors fall back to their own names.
'datasource.connector.',
'datasource.connectorDesc.',
] as const ] as const
/** Keys that must survive pruning even when static analysis misses them. */ /** Keys that must survive pruning even when static analysis misses them. */

View File

@ -812,6 +812,8 @@ export default {
channelGitLab: 'GitLab', channelGitLab: 'GitLab',
channelIma: 'Tencent IMA', channelIma: 'Tencent IMA',
channelUpload: 'Upload', channelUpload: 'Upload',
channelRss: 'RSS',
channelExternal: 'External source',
channelManual: 'Manual', channelManual: 'Manual',
channelUrl: 'Web', channelUrl: 'Web',
channelUnknown: 'Unknown', channelUnknown: 'Unknown',
@ -2732,6 +2734,10 @@ export default {
} }
}, },
pluginAdmin: { pluginAdmin: {
runtime: {
declarative: 'Declarative',
host: 'Local process'
},
title: 'Plugin management', title: 'Plugin management',
description: 'Install and manage plugins for the whole platform. Every workspace sees an installed plugin, and each one enables it for itself; disabling a plugin here unloads it on every node.', description: 'Install and manage plugins for the whole platform. Every workspace sees an installed plugin, and each one enables it for itself; disabling a plugin here unloads it on every node.',
installButton: 'Install plugin', installButton: 'Install plugin',
@ -2745,6 +2751,7 @@ export default {
contributions: 'What it adds', contributions: 'What it adds',
permissions: 'Permissions and remote access', permissions: 'Permissions and remote access',
state: { state: {
degraded: 'Restarting',
running: 'Running', running: 'Running',
failed: 'Failed to load', failed: 'Failed to load',
disabled: 'Disabled', disabled: 'Disabled',
@ -2767,6 +2774,7 @@ export default {
reinstall: 'Reinstall' reinstall: 'Reinstall'
}, },
permission: { permission: {
anyHost: 'Any public host',
remote: 'Remote service', remote: 'Remote service',
egress: 'Network egress', egress: 'Network egress',
hostApi: 'WeKnora API', hostApi: 'WeKnora API',

View File

@ -812,6 +812,8 @@ export default {
channelGitLab: 'GitLab', channelGitLab: 'GitLab',
channelIma: 'Tencent IMA', channelIma: 'Tencent IMA',
channelUpload: 'アップロード', channelUpload: 'アップロード',
channelRss: 'RSS',
channelExternal: '外部データソース',
channelManual: '手動作成', channelManual: '手動作成',
channelUrl: 'Web', channelUrl: 'Web',
channelUnknown: '不明', channelUnknown: '不明',
@ -2732,6 +2734,10 @@ export default {
} }
}, },
pluginAdmin: { pluginAdmin: {
runtime: {
declarative: '宣言型',
host: 'ローカルプロセス'
},
title: 'プラグイン管理', title: 'プラグイン管理',
description: 'プラットフォーム全体のプラグインをインストール・管理します。インストールしたプラグインはすべてのワークスペースに表示され、各ワークスペースで個別に有効化します。ここで無効にすると全ノードでアンロードされます。', description: 'プラットフォーム全体のプラグインをインストール・管理します。インストールしたプラグインはすべてのワークスペースに表示され、各ワークスペースで個別に有効化します。ここで無効にすると全ノードでアンロードされます。',
installButton: 'プラグインをインストール', installButton: 'プラグインをインストール',
@ -2745,6 +2751,7 @@ export default {
contributions: '提供する機能', contributions: '提供する機能',
permissions: '権限と外部アクセス', permissions: '権限と外部アクセス',
state: { state: {
degraded: '再起動中',
running: '稼働中', running: '稼働中',
failed: '読み込み失敗', failed: '読み込み失敗',
disabled: '無効', disabled: '無効',
@ -2767,6 +2774,7 @@ export default {
reinstall: '再インストール' reinstall: '再インストール'
}, },
permission: { permission: {
anyHost: '任意の公開ホスト',
remote: 'リモートサービス', remote: 'リモートサービス',
egress: 'ネットワーク送信', egress: 'ネットワーク送信',
hostApi: 'WeKnora API', hostApi: 'WeKnora API',

View File

@ -5252,6 +5252,10 @@ export default {
noCompatibleKbForAgent: '현재 에이전트의 도구와 범위 내 지식베이스의 기능이 일치하지 않아 참조할 수 있는 지식베이스가 없습니다.' noCompatibleKbForAgent: '현재 에이전트의 도구와 범위 내 지식베이스의 기능이 일치하지 않아 참조할 수 있는 지식베이스가 없습니다.'
}, },
pluginAdmin: { pluginAdmin: {
runtime: {
declarative: '선언형',
host: '로컬 프로세스'
},
title: '플러그인 관리', title: '플러그인 관리',
description: '플랫폼 전체의 플러그인을 설치하고 관리합니다. 설치된 플러그인은 모든 워크스페이스에 보이며 각 워크스페이스가 직접 활성화합니다. 여기서 비활성화하면 모든 노드에서 언로드됩니다.', description: '플랫폼 전체의 플러그인을 설치하고 관리합니다. 설치된 플러그인은 모든 워크스페이스에 보이며 각 워크스페이스가 직접 활성화합니다. 여기서 비활성화하면 모든 노드에서 언로드됩니다.',
installButton: '플러그인 설치', installButton: '플러그인 설치',
@ -5265,6 +5269,7 @@ export default {
contributions: '제공 기능', contributions: '제공 기능',
permissions: '권한 및 외부 접근', permissions: '권한 및 외부 접근',
state: { state: {
degraded: '재시작 중',
running: '실행 중', running: '실행 중',
failed: '로드 실패', failed: '로드 실패',
disabled: '비활성', disabled: '비활성',
@ -5287,6 +5292,7 @@ export default {
reinstall: '재설치' reinstall: '재설치'
}, },
permission: { permission: {
anyHost: '모든 공개 호스트',
remote: '원격 서비스', remote: '원격 서비스',
egress: '네트워크 송신', egress: '네트워크 송신',
hostApi: 'WeKnora API', hostApi: 'WeKnora API',
@ -7315,6 +7321,8 @@ export default {
channelGitLab: 'GitLab', channelGitLab: 'GitLab',
channelIma: 'Tencent IMA', channelIma: 'Tencent IMA',
channelUpload: '업로드', channelUpload: '업로드',
channelRss: 'RSS',
channelExternal: '외부 데이터 소스',
channelManual: '수동', channelManual: '수동',
channelUrl: '웹', channelUrl: '웹',
channelUnknown: '알 수 없음', channelUnknown: '알 수 없음',

View File

@ -5252,6 +5252,10 @@ export default {
noCompatibleKbForAgent: 'Инструменты текущего агента не соответствуют возможностям ни одной базы знаний в области видимости — нечего упомянуть.' noCompatibleKbForAgent: 'Инструменты текущего агента не соответствуют возможностям ни одной базы знаний в области видимости — нечего упомянуть.'
}, },
pluginAdmin: { pluginAdmin: {
runtime: {
declarative: 'Декларативный',
host: 'Локальный процесс'
},
title: 'Управление плагинами', title: 'Управление плагинами',
description: 'Установка и управление плагинами всей платформы. Установленный плагин виден всем рабочим пространствам, и каждое включает его самостоятельно; отключение здесь выгружает плагин на всех узлах.', description: 'Установка и управление плагинами всей платформы. Установленный плагин виден всем рабочим пространствам, и каждое включает его самостоятельно; отключение здесь выгружает плагин на всех узлах.',
installButton: 'Установить плагин', installButton: 'Установить плагин',
@ -5265,6 +5269,7 @@ export default {
contributions: 'Что добавляет', contributions: 'Что добавляет',
permissions: 'Разрешения и внешний доступ', permissions: 'Разрешения и внешний доступ',
state: { state: {
degraded: 'Перезапуск',
running: 'Работает', running: 'Работает',
failed: 'Ошибка загрузки', failed: 'Ошибка загрузки',
disabled: 'Отключён', disabled: 'Отключён',
@ -5287,6 +5292,7 @@ export default {
reinstall: 'Переустановка' reinstall: 'Переустановка'
}, },
permission: { permission: {
anyHost: 'Любой публичный хост',
remote: 'Удалённый сервис', remote: 'Удалённый сервис',
egress: 'Исходящая сеть', egress: 'Исходящая сеть',
hostApi: 'API WeKnora', hostApi: 'API WeKnora',
@ -7315,6 +7321,8 @@ export default {
channelGitLab: 'GitLab', channelGitLab: 'GitLab',
channelIma: 'Tencent IMA', channelIma: 'Tencent IMA',
channelUpload: 'Загрузка', channelUpload: 'Загрузка',
channelRss: 'RSS',
channelExternal: 'Внешний источник',
channelManual: 'Вручную', channelManual: 'Вручную',
channelUrl: 'Веб', channelUrl: 'Веб',
channelUnknown: 'Неизвестно', channelUnknown: 'Неизвестно',

View File

@ -5254,6 +5254,10 @@ export default {
noCompatibleKbForAgent: '当前智能体的工具与作用域内知识库的能力不匹配,暂无可引用的知识库。' noCompatibleKbForAgent: '当前智能体的工具与作用域内知识库的能力不匹配,暂无可引用的知识库。'
}, },
pluginAdmin: { pluginAdmin: {
runtime: {
declarative: '声明式',
host: '本机进程'
},
title: '插件管理', title: '插件管理',
description: '安装与管理本平台的插件。安装后所有空间都能看到,但每个空间需自行启用;停用会在所有节点卸载该插件。', description: '安装与管理本平台的插件。安装后所有空间都能看到,但每个空间需自行启用;停用会在所有节点卸载该插件。',
installButton: '安装插件', installButton: '安装插件',
@ -5267,6 +5271,7 @@ export default {
contributions: '提供的能力', contributions: '提供的能力',
permissions: '权限与外部访问', permissions: '权限与外部访问',
state: { state: {
degraded: '重启中',
running: '运行中', running: '运行中',
failed: '加载失败', failed: '加载失败',
disabled: '已停用', disabled: '已停用',
@ -5289,6 +5294,7 @@ export default {
reinstall: '重新安装' reinstall: '重新安装'
}, },
permission: { permission: {
anyHost: '任意公网地址',
remote: '访问远程服务', remote: '访问远程服务',
egress: '网络出口', egress: '网络出口',
hostApi: '调用 WeKnora API', hostApi: '调用 WeKnora API',
@ -7317,6 +7323,8 @@ export default {
channelGitLab: 'GitLab', channelGitLab: 'GitLab',
channelIma: '腾讯 IMA', channelIma: '腾讯 IMA',
channelUpload: '上传', channelUpload: '上传',
channelRss: 'RSS 订阅',
channelExternal: '外部数据源',
channelManual: '手动', channelManual: '手动',
channelUrl: '网页', channelUrl: '网页',
channelUnknown: '未知', channelUnknown: '未知',

View File

@ -9,3 +9,18 @@ export function localizedText(text: LocalizedText | undefined, locale: string):
const match = Object.keys(text).sort().find((k) => k !== 'default' && k.split('-')[0] === lang && text[k]) const match = Object.keys(text).sort().find((k) => k !== 'default' && k.split('-')[0] === lang && text[k])
return match ? text[match] : text.default return match ? text[match] : text.default
} }
/**
* Picks a variant from a locale map without a default ({ 'zh-CN': ... }, as
* plugin connectors and provider types send names): exact, then same
* language, else undefined.
*/
export function pickLocale(variants: Record<string, string> | undefined, locale: string): string | undefined {
if (!variants) return undefined
if (variants[locale]) return variants[locale]
const lang = locale.split('-')[0].toLowerCase()
const key = Object.keys(variants)
.sort()
.find((k) => k.split('-')[0].toLowerCase() === lang)
return key ? variants[key] : undefined
}

View File

@ -109,6 +109,9 @@ const getSourceInfo = (item: KnowledgeItem): { icon: string; label: string } =>
if (ch === 'dingtalk') return { icon: 'cloud-download', label: t('knowledgeBase.channelDingtalk') }; if (ch === 'dingtalk') return { icon: 'cloud-download', label: t('knowledgeBase.channelDingtalk') };
if (ch === 'slack') return { icon: 'cloud-download', label: t('knowledgeBase.channelSlack') }; if (ch === 'slack') return { icon: 'cloud-download', label: t('knowledgeBase.channelSlack') };
if (ch === 'im') return { icon: 'cloud-download', label: t('knowledgeBase.channelIm') }; if (ch === 'im') return { icon: 'cloud-download', label: t('knowledgeBase.channelIm') };
if (ch === 'rss') return { icon: 'cloud-download', label: t('knowledgeBase.channelRss') };
// Plugin connectors ingest under their qualified type ID (acme.feeds/feed).
if (ch && ch.includes('/')) return { icon: 'cloud-download', label: t('knowledgeBase.channelExternal') };
if (item.type === 'url') return { icon: 'link', label: t('knowledgeBase.channelUrl') }; if (item.type === 'url') return { icon: 'link', label: t('knowledgeBase.channelUrl') };
if (item.type === 'manual') return { icon: 'edit', label: t('knowledgeBase.channelManual') }; if (item.type === 'manual') return { icon: 'edit', label: t('knowledgeBase.channelManual') };
return { icon: 'upload', label: t('knowledgeBase.channelUpload') }; return { icon: 'upload', label: t('knowledgeBase.channelUpload') };

View File

@ -80,7 +80,15 @@ async function fixture({
exports, exports,
require(name: string) { require(name: string) {
if (name === 'vue') return require('vue') if (name === 'vue') return require('vue')
if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key }) } if (name === 'vue-i18n') {
return { useI18n: () => ({ t: (key: string) => key, te: () => true, locale: { value: 'en-US' } }) }
}
if (name === './connectorLabels') {
return {
connectorName: (d: { type: string }) => `datasource.connector.${d.type}`,
connectorDescription: (d: { type: string }) => `datasource.connectorDesc.${d.type}`,
}
}
if (name === 'tdesign-vue-next') return { MessagePlugin: { warning() {}, success() {}, error() {} } } if (name === 'tdesign-vue-next') return { MessagePlugin: { warning() {}, success() {}, error() {} } }
if (name === '@/api/datasource') return api if (name === '@/api/datasource') return api
if (name === '@/components/schema-form/schema') return schemaModule if (name === '@/components/schema-form/schema') return schemaModule

View File

@ -29,6 +29,7 @@ import {
import { useSchemaText } from '@/components/schema-form/useSchemaText' import { useSchemaText } from '@/components/schema-form/useSchemaText'
import SettingDrawer from '@/components/settings/SettingDrawer.vue' import SettingDrawer from '@/components/settings/SettingDrawer.vue'
import DataSourceTypeIcon from './DataSourceTypeIcon.vue' import DataSourceTypeIcon from './DataSourceTypeIcon.vue'
import { connectorDescription, connectorName } from './connectorLabels'
import { getDatasourceIconUrl } from './datasourceIcons' import { getDatasourceIconUrl } from './datasourceIcons'
const props = defineProps<{ const props = defineProps<{
@ -38,7 +39,7 @@ const props = defineProps<{
const visible = defineModel<boolean>('visible', { default: false }) const visible = defineModel<boolean>('visible', { default: false })
const emit = defineEmits<{ saved: [] }>() const emit = defineEmits<{ saved: [] }>()
const { t } = useI18n() const { t, te, locale } = useI18n()
const isEdit = computed(() => !!props.dataSource) const isEdit = computed(() => !!props.dataSource)
const step = ref(0) const step = ref(0)
@ -536,6 +537,11 @@ const currentDef = computed(() => connectorTypes.value.find(d => d.type === form
const EMPTY_SCHEMA: ConfigSchema = { type: 'object', properties: {} } const EMPTY_SCHEMA: ConfigSchema = { type: 'object', properties: {} }
const credentialSchema = computed<ConfigSchema>(() => currentDef.value?.config_schema ?? EMPTY_SCHEMA) const credentialSchema = computed<ConfigSchema>(() => currentDef.value?.config_schema ?? EMPTY_SCHEMA)
// Settings form of connectors without a built-in settings UI (plugins).
const settingsSchema = computed<ConfigSchema | null>(() => currentDef.value?.settings_schema ?? null)
const settingsErrors = ref<FieldError[]>([])
const labelI18n = computed(() => ({ t: (k: string) => t(k), te: (k: string) => te(k), locale: locale.value }))
const labelOf = (def: ConnectorMeta) => connectorName(def, labelI18n.value)
// Field errors of the last credential check, shown under each field. // Field errors of the last credential check, shown under each field.
const credentialErrors = ref<FieldError[]>([]) const credentialErrors = ref<FieldError[]>([])
@ -547,9 +553,12 @@ watch(() => form.value.config.credentials, () => { credentialErrors.value = [] }
// fields and warns with the first one, as the editor always has. // fields and warns with the first one, as the editor always has.
function checkCredentials(): boolean { function checkCredentials(): boolean {
credentialErrors.value = validateConfig(credentialSchema.value, form.value.config.credentials) credentialErrors.value = validateConfig(credentialSchema.value, form.value.config.credentials)
const first = credentialErrors.value[0] settingsErrors.value = settingsSchema.value ? validateConfig(settingsSchema.value, form.value.config.settings) : []
const inCredentials = credentialErrors.value.length > 0
const first = credentialErrors.value[0] ?? settingsErrors.value[0]
if (!first) return true if (!first) return true
const field = schemaAt(credentialSchema.value, first.path) const schema = inCredentials ? credentialSchema.value : (settingsSchema.value as ConfigSchema)
const field = schemaAt(schema, first.path)
const label = field ? schemaText(field, 'title') : first.path const label = field ? schemaText(field, 'title') : first.path
MessagePlugin.warning( MessagePlugin.warning(
first.code === 'required' first.code === 'required'
@ -673,9 +682,13 @@ watch(
function selectType(def: ConnectorMeta) { function selectType(def: ConnectorMeta) {
form.value.type = def.type form.value.type = def.type
form.value.name = t(`datasource.connector.${def.type}`) form.value.name = labelOf(def)
// Schema defaults, e.g. Confluence starts on the Server edition. // Schema defaults, e.g. Confluence starts on the Server edition.
form.value.config.credentials = applyDefaults(def.config_schema ?? EMPTY_SCHEMA, {}) form.value.config.credentials = applyDefaults(def.config_schema ?? EMPTY_SCHEMA, {})
if (def.settings_schema) {
form.value.config.settings = applyDefaults(def.settings_schema, form.value.config.settings ?? {})
}
settingsErrors.value = []
credentialErrors.value = [] credentialErrors.value = []
if (def.type === 'confluence') { if (def.type === 'confluence') {
form.value.config.settings = { ...form.value.config.settings, edition: 'server' } form.value.config.settings = { ...form.value.config.settings, edition: 'server' }
@ -717,7 +730,7 @@ async function testConnection() {
// validate-credentials is credentials-only; feed URLs live in settings. // validate-credentials is credentials-only; feed URLs live in settings.
creds.feed_urls = form.value.config.settings.feed_urls creds.feed_urls = form.value.config.settings.feed_urls
} }
await validateCredentials(form.value.type, creds) await validateCredentials(form.value.type, creds, form.value.config.settings)
} }
testResult.value = 'success' testResult.value = 'success'
MessagePlugin.success(t('datasource.testSuccess')) MessagePlugin.success(t('datasource.testSuccess'))
@ -1137,9 +1150,9 @@ const drawerConfirmText = computed(() => {
@confirm="handleDrawerConfirm" @confirm="handleDrawerConfirm"
@cancel="handleClose" @cancel="handleClose"
> >
<template v-if="form.type && getDatasourceIconUrl(form.type)" #headerIcon> <template v-if="form.type && (getDatasourceIconUrl(form.type) || currentDef?.icon)" #headerIcon>
<img <img
:src="getDatasourceIconUrl(form.type)" :src="getDatasourceIconUrl(form.type) || currentDef?.icon"
:alt="form.type" :alt="form.type"
class="datasource-header-icon__img" class="datasource-header-icon__img"
> >
@ -1210,10 +1223,10 @@ const drawerConfirmText = computed(() => {
@click="selectType(def)" @click="selectType(def)"
> >
<div class="ds-type-header"> <div class="ds-type-header">
<DataSourceTypeIcon :type="def.type" :size="20" /> <DataSourceTypeIcon :type="def.type" :icon-url="def.icon" :label="labelOf(def)" :size="20" />
<span class="ds-type-name">{{ t(`datasource.connector.${def.type}`) }}</span> <span class="ds-type-name">{{ labelOf(def) }}</span>
</div> </div>
<div class="ds-type-desc">{{ t(`datasource.connectorDesc.${def.type}`) }}</div> <div class="ds-type-desc">{{ connectorDescription(def, labelI18n) }}</div>
</button> </button>
</div> </div>
</section> </section>
@ -1388,6 +1401,12 @@ const drawerConfirmText = computed(() => {
:schema="credentialSchema" :schema="credentialSchema"
:errors="credentialErrors" :errors="credentialErrors"
/> />
<SchemaForm
v-if="settingsSchema"
v-model="form.config.settings"
:schema="settingsSchema"
:errors="settingsErrors"
/>
<div v-if="isEdit && replaceCredentialsMode" class="credential-edit-actions"> <div v-if="isEdit && replaceCredentialsMode" class="credential-edit-actions">
<t-button size="small" variant="text" @click="cancelReplaceCredentials"> <t-button size="small" variant="text" @click="cancelReplaceCredentials">
{{ t('common.cancel') }} {{ t('common.cancel') }}

View File

@ -3,22 +3,36 @@ import { ref, computed, onMounted, onBeforeUnmount } from 'vue'
import { MessagePlugin } from 'tdesign-vue-next' import { MessagePlugin } from 'tdesign-vue-next'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
import { import {
getConnectorTypes,
listDataSources, listDataSources,
deleteDataSource, deleteDataSource,
triggerSync, triggerSync,
pauseDataSource, pauseDataSource,
resumeDataSource, resumeDataSource,
type ConnectorMeta,
type DataSource, type DataSource,
} from '@/api/datasource' } from '@/api/datasource'
import { humanizeCron, relativeTime } from '@/utils/cronHumanize' import { humanizeCron, relativeTime } from '@/utils/cronHumanize'
import DataSourceEditorDialog from './DataSourceEditorDialog.vue' import DataSourceEditorDialog from './DataSourceEditorDialog.vue'
import DataSourceSyncLogs from './DataSourceSyncLogs.vue' import DataSourceSyncLogs from './DataSourceSyncLogs.vue'
import DataSourceTypeIcon from './DataSourceTypeIcon.vue' import DataSourceTypeIcon from './DataSourceTypeIcon.vue'
import { connectorName } from './connectorLabels'
import { useAuthStore } from '@/stores/auth' import { useAuthStore } from '@/stores/auth'
const props = defineProps<{ kbId: string }>() const props = defineProps<{ kbId: string }>()
const emit = defineEmits<{ (e: 'count', value: number): void }>() const emit = defineEmits<{ (e: 'count', value: number): void }>()
const { t } = useI18n() const { t, te, locale } = useI18n()
// Connector metadata names plugin connectors, which have no locale keys.
const connectorMeta = ref<Record<string, ConnectorMeta>>({})
async function loadConnectorMeta() {
try {
const list = await getConnectorTypes()
connectorMeta.value = Object.fromEntries([...list].map((m) => [m.type, m]))
} catch {
connectorMeta.value = {}
}
}
const authStore = useAuthStore() const authStore = useAuthStore()
// 后端 /datasource 的 list/logs 是 Viewer+,但所有写操作(POST/PUT/DELETE // 后端 /datasource 的 list/logs 是 Viewer+,但所有写操作(POST/PUT/DELETE
@ -134,7 +148,7 @@ function syncModeLabel(mode: string) {
} }
function connectorLabel(type: string) { function connectorLabel(type: string) {
return t(`datasource.connector.${type}`) || type return connectorName(connectorMeta.value[type] ?? { type }, { t, te, locale: locale.value })
} }
function scheduleLabel(cron: string) { function scheduleLabel(cron: string) {
@ -177,7 +191,10 @@ function onEditorSaved() {
loadList() loadList()
} }
onMounted(loadList) onMounted(() => {
void loadConnectorMeta()
void loadList()
})
onBeforeUnmount(stopPolling) onBeforeUnmount(stopPolling)
</script> </script>
@ -206,7 +223,12 @@ onBeforeUnmount(stopPolling)
@click="canManageDataSource ? openEdit(ds) : undefined" @click="canManageDataSource ? openEdit(ds) : undefined"
> >
<div class="ds-card__badge"> <div class="ds-card__badge">
<DataSourceTypeIcon :type="ds.type" variant="badge" /> <DataSourceTypeIcon
:type="ds.type"
:icon-url="connectorMeta[ds.type]?.icon"
:label="connectorLabel(ds.type)"
variant="badge"
/>
</div> </div>
<div class="ds-card__body"> <div class="ds-card__body">
<div class="ds-card__header"> <div class="ds-card__header">

View File

@ -3,10 +3,16 @@ import { getDatasourceIconUrl, datasourceIconMap } from './datasourceIcons'
const props = withDefaults(defineProps<{ const props = withDefaults(defineProps<{
type: string type: string
/** Icon of a plugin connector (a data: URI from its metadata). */
iconUrl?: string
/** Display name, whose first letter stands in when there is no icon. */
label?: string
size?: number size?: number
/** inline: 类型选择等小尺寸场景;badge: 嵌入 ds-card__badge 等父级徽章容器 */ /** inline: 类型选择等小尺寸场景;badge: 嵌入 ds-card__badge 等父级徽章容器 */
variant?: 'inline' | 'badge' variant?: 'inline' | 'badge'
}>(), { }>(), {
iconUrl: '',
label: '',
size: 20, size: 20,
variant: 'inline', variant: 'inline',
}) })
@ -26,7 +32,7 @@ function fallbackText(type: string) {
case 'ima': case 'ima':
return 'I' return 'I'
default: default:
return type.slice(0, 1).toUpperCase() || '?' return (props.label || type).trim().slice(0, 1).toUpperCase() || '?'
} }
} }
</script> </script>
@ -44,6 +50,13 @@ function fallbackText(type: string) {
class="ds-type-icon__img" class="ds-type-icon__img"
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined" :style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
> >
<img
v-else-if="iconUrl"
:src="iconUrl"
:alt="label || type"
class="ds-type-icon__img"
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
>
<span v-else class="ds-type-icon-fallback">{{ fallbackText(type) }}</span> <span v-else class="ds-type-icon-fallback">{{ fallbackText(type) }}</span>
</span> </span>
</template> </template>

View File

@ -0,0 +1,33 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { connectorDescription, connectorName, isPluginChannel, pickLocale } from './connectorLabels'
const keys: Record<string, string> = { 'datasource.connector.rss': 'RSS', 'datasource.connectorDesc.rss': 'Feeds' }
const i18n = (locale: string) => ({ t: (k: string) => keys[k] ?? k, te: (k: string) => k in keys, locale })
test('builtins use their locale keys', () => {
assert.equal(connectorName({ type: 'rss', name: 'ignored' }, i18n('zh-CN')), 'RSS')
assert.equal(connectorDescription({ type: 'rss' }, i18n('en-US')), 'Feeds')
})
test('plugin connectors use their own localized names', () => {
const plugin = {
type: 'acme.feeds/feed',
name: 'ACME Feed',
names: { 'en-US': 'ACME Feed', 'zh-CN': 'ACME 订阅' },
description: 'Feeds',
}
assert.equal(connectorName(plugin, i18n('zh-CN')), 'ACME 订阅')
assert.equal(connectorName(plugin, i18n('zh-TW')), 'ACME 订阅')
assert.equal(connectorName(plugin, i18n('ja-JP')), 'ACME Feed')
assert.equal(connectorDescription(plugin, i18n('ja-JP')), 'Feeds')
assert.equal(connectorName({ type: 'acme.x/y' }, i18n('en-US')), 'acme.x/y')
})
test('pickLocale and plugin channels', () => {
assert.equal(pickLocale({ 'zh-CN': 'a' }, 'zh-HK'), 'a')
assert.equal(pickLocale(undefined, 'en-US'), undefined)
assert.equal(isPluginChannel('acme.feeds/feed'), true)
assert.equal(isPluginChannel('rss'), false)
})

View File

@ -0,0 +1,41 @@
// Display names of data source connectors. Builtins have locale keys
// (datasource.connector.<type>); plugin connectors bring localized names in
// their metadata. Kept free of Vue so it runs under node:test.
import { pickLocale } from '../../../utils/localizedText'
export { pickLocale }
/** The parts of connector metadata labels need. */
export interface ConnectorLabelSource {
type: string
name?: string
description?: string
names?: Record<string, string>
descriptions?: Record<string, string>
}
/** The subset of vue-i18n the helpers use. */
export interface LabelI18n {
t: (key: string) => string
te: (key: string) => boolean
locale: string
}
/** A connector's name: locale key, plugin variant, plugin default, type. */
export function connectorName(src: ConnectorLabelSource, i18n: LabelI18n): string {
const key = `datasource.connector.${src.type}`
if (i18n.te(key)) return i18n.t(key)
return pickLocale(src.names, i18n.locale) || src.name || src.type
}
/** A connector's description, the same way; empty when there is none. */
export function connectorDescription(src: ConnectorLabelSource, i18n: LabelI18n): string {
const key = `datasource.connectorDesc.${src.type}`
if (i18n.te(key)) return i18n.t(key)
return pickLocale(src.descriptions, i18n.locale) || src.description || ''
}
/** Channels plugin connectors ingest under: their qualified type ID. */
export function isPluginChannel(channel: string | undefined): boolean {
return !!channel && channel.includes('/')
}

View File

@ -288,9 +288,10 @@ import CredentialResource, {
import { useConfirmDelete } from '@/components/settings/useConfirmDelete' import { useConfirmDelete } from '@/components/settings/useConfirmDelete'
import { useAuthStore } from '@/stores/auth' import { useAuthStore } from '@/stores/auth'
import { useChatResourcesStore } from '@/stores/chatResources' import { useChatResourcesStore } from '@/stores/chatResources'
import { pickLocale } from '@/utils/localizedText'
import { providerLogo } from './providerLogos' import { providerLogo } from './providerLogos'
const { t } = useI18n() const { t, locale } = useI18n()
const authStore = useAuthStore() const authStore = useAuthStore()
const chatResources = useChatResourcesStore() const chatResources = useChatResourcesStore()
const confirmDelete = useConfirmDelete() const confirmDelete = useConfirmDelete()
@ -398,7 +399,7 @@ const drawerClass = computed(() => {
// list card showed for the same provider id. // list card showed for the same provider id.
const drawerLogo = computed(() => { const drawerLogo = computed(() => {
const id = providerForm.value.provider const id = providerForm.value.provider
return id ? providerLogo('websearch', id) : null return id ? resolveLogo(id) ?? null : null
}) })
const drawerLogoStyle = computed((): Record<string, string> => { const drawerLogoStyle = computed((): Record<string, string> => {
@ -424,7 +425,13 @@ const providerInitial = (providerId: string) => {
} }
// 见 VectorStoreSettings 的同名注释:返回 --logo-url 给 ::before 用 mask 渲染。 // 见 VectorStoreSettings 的同名注释:返回 --logo-url 给 ::before 用 mask 渲染。
const resolveLogo = (providerId: string) => providerLogo('websearch', providerId) // Plugin provider types carry their own icon.
const resolveLogo = (providerId: string) => {
const bundled = providerLogo('websearch', providerId)
if (bundled) return bundled
const icon = providerTypes.value.find((p) => p.id === providerId)?.icon
return icon ? { mode: 'color' as const, url: icon } : undefined
}
const badgeClass = (providerId: string) => { const badgeClass = (providerId: string) => {
const m = resolveLogo(providerId)?.mode const m = resolveLogo(providerId)?.mode
@ -470,7 +477,11 @@ const loadProviderEntities = async () => {
const loadProviderTypes = async () => { const loadProviderTypes = async () => {
try { try {
providerTypes.value = await listWebSearchProviderTypes() // Plugin provider types name themselves per locale.
providerTypes.value = (await listWebSearchProviderTypes()).map((pt) => ({
...pt,
name: pickLocale(pt.names, locale.value) || pt.name,
}))
} catch (error) { } catch (error) {
console.error('Failed to load provider types:', error) console.error('Failed to load provider types:', error)
} }

View File

@ -42,7 +42,7 @@
</t-tag> </t-tag>
</t-tooltip> </t-tooltip>
</div> </div>
<div class="plugin-card__meta">{{ p.id }} · v{{ p.active_version }} · {{ p.runtime }}</div> <div class="plugin-card__meta">{{ p.id }} · v{{ p.active_version }} · {{ runtimeLabel(p.runtime) }}</div>
<div v-if="descriptionOf(p)" class="plugin-card__desc">{{ descriptionOf(p) }}</div> <div v-if="descriptionOf(p)" class="plugin-card__desc">{{ descriptionOf(p) }}</div>
<div v-if="p.manifest" class="plugin-card__contribs"> <div v-if="p.manifest" class="plugin-card__contribs">
<span v-for="s in contributionSummary(p.manifest)" :key="s.point" class="plugin-card__contrib"> <span v-for="s in contributionSummary(p.manifest)" :key="s.point" class="plugin-card__contrib">
@ -89,7 +89,8 @@ import { installedState } from './pluginManagementState'
// Platform plugin management: what is installed, on which version, whether it // Platform plugin management: what is installed, on which version, whether it
// loads. Installing makes a plugin available to every workspace; each // loads. Installing makes a plugin available to every workspace; each
// workspace still turns it on in its own plugin center. // workspace still turns it on in its own plugin center.
const { t, locale } = useI18n() const { t, te, locale } = useI18n()
const runtimeLabel = (rt: string) => (te(`pluginAdmin.runtime.${rt}`) ? t(`pluginAdmin.runtime.${rt}`) : rt)
const plugins = ref<InstalledPlugin[]>([]) const plugins = ref<InstalledPlugin[]>([])
const loading = ref(false) const loading = ref(false)
@ -109,6 +110,8 @@ function stateTheme(p: InstalledPlugin) {
return 'success' return 'success'
case 'failed': case 'failed':
return 'danger' return 'danger'
case 'degraded':
return 'warning'
default: default:
return 'default' return 'default'
} }

View File

@ -67,6 +67,7 @@ test('installedState reads desired state and the node report', () => {
assert.equal(installedState(base), 'pending') assert.equal(installedState(base), 'pending')
assert.equal(installedState({ ...base, node: { version: '1', state: 'ready', updatedAt: '' } }), 'running') assert.equal(installedState({ ...base, node: { version: '1', state: 'ready', updatedAt: '' } }), 'running')
assert.equal(installedState({ ...base, node: { version: '1', state: 'failed', updatedAt: '' } }), 'failed') assert.equal(installedState({ ...base, node: { version: '1', state: 'failed', updatedAt: '' } }), 'failed')
assert.equal(installedState({ ...base, node: { version: '1', state: 'degraded', updatedAt: '' } }), 'degraded')
}) })
test('isPackageUrl and hasSystemConfig', () => { test('isPackageUrl and hasSystemConfig', () => {

View File

@ -95,14 +95,24 @@ export function sortVersions(list: readonly PluginVersion[]): PluginVersion[] {
} }
/** Overall state of an installed plugin on the node that answered. */ /** Overall state of an installed plugin on the node that answered. */
export type InstalledState = 'running' | 'failed' | 'disabled' | 'pending' export type InstalledState = 'running' | 'degraded' | 'failed' | 'disabled' | 'pending'
export function installedState(p: InstalledPlugin): InstalledState { export function installedState(p: InstalledPlugin): InstalledState {
if (p.desired_state === 'disabled') return 'disabled' if (p.desired_state === 'disabled') return 'disabled'
if (!p.node) return 'pending' if (!p.node) return 'pending'
return p.node.state === 'ready' ? 'running' : 'failed' switch (p.node.state) {
case 'ready':
return 'running'
case 'degraded':
return 'degraded'
default:
return 'failed'
}
} }
/** The egress grant meaning any public host. */
export const EGRESS_ANY_HOST = '*'
/** Whether a string can be sent as a package URL. */ /** Whether a string can be sent as a package URL. */
export function isPackageUrl(raw: string): boolean { export function isPackageUrl(raw: string): boolean {
try { try {

View File

@ -19,7 +19,7 @@
<dt>{{ t('pluginAdmin.publisher') }}</dt> <dt>{{ t('pluginAdmin.publisher') }}</dt>
<dd>{{ manifest?.publisher.name || manifest?.publisher.id }}</dd> <dd>{{ manifest?.publisher.name || manifest?.publisher.id }}</dd>
<dt>{{ t('pluginAdmin.detail.runtime') }}</dt> <dt>{{ t('pluginAdmin.detail.runtime') }}</dt>
<dd>{{ plugin.runtime }}</dd> <dd>{{ te(`pluginAdmin.runtime.${plugin.runtime}`) ? t(`pluginAdmin.runtime.${plugin.runtime}`) : plugin.runtime }}</dd>
<dt>{{ t('pluginAdmin.detail.source') }}</dt> <dt>{{ t('pluginAdmin.detail.source') }}</dt>
<dd> <dd>
{{ t(`pluginAdmin.source.${plugin.source?.kind ?? 'upload'}`) }} {{ t(`pluginAdmin.source.${plugin.source?.kind ?? 'upload'}`) }}
@ -141,7 +141,7 @@ const emit = defineEmits<{
removed: [id: string] removed: [id: string]
}>() }>()
const { t, locale } = useI18n() const { t, te, locale } = useI18n()
const manifest = computed(() => props.plugin?.manifest) const manifest = computed(() => props.plugin?.manifest)
const title = computed(() => (manifest.value ? localizedText(manifest.value.name, locale.value) : props.plugin?.id ?? '')) const title = computed(() => (manifest.value ? localizedText(manifest.value.name, locale.value) : props.plugin?.id ?? ''))

View File

@ -96,7 +96,10 @@
</li> </li>
<li v-for="p in permissions" :key="`${p.kind}:${p.value}`"> <li v-for="p in permissions" :key="`${p.kind}:${p.value}`">
<span class="review-list__point">{{ t(`pluginAdmin.permission.${p.kind}`) }}</span> <span class="review-list__point">{{ t(`pluginAdmin.permission.${p.kind}`) }}</span>
<code class="review-list__detail">{{ p.value }}</code> <strong v-if="p.kind === 'egress' && p.value === EGRESS_ANY_HOST" class="review-list__warn">
{{ t('pluginAdmin.permission.anyHost') }}
</strong>
<code v-else class="review-list__detail">{{ p.value }}</code>
</li> </li>
</ul> </ul>
</div> </div>
@ -128,7 +131,15 @@ import {
} from '@/api/system/plugins' } from '@/api/system/plugins'
import { localizedText } from '@/utils/localizedText' import { localizedText } from '@/utils/localizedText'
import { hasSystemConfig, contributionLines, formatBytes, isPackageUrl, permissionLines, remoteHosts } from '../pluginManagementState' import {
EGRESS_ANY_HOST,
contributionLines,
formatBytes,
hasSystemConfig,
isPackageUrl,
permissionLines,
remoteHosts,
} from '../pluginManagementState'
import { hasTenantConfig } from '../../settings/pluginCenterState' import { hasTenantConfig } from '../../settings/pluginCenterState'
// Installing is two steps on purpose: inspect shows what the package would // Installing is two steps on purpose: inspect shows what the package would
@ -393,6 +404,12 @@ async function onConfirm() {
color: var(--td-text-color-placeholder); color: var(--td-text-color-placeholder);
word-break: break-all; word-break: break-all;
} }
&__warn {
font-size: var(--app-text-sm);
font-weight: 500;
color: var(--td-warning-color);
}
} }
.review-alert { .review-alert {

3
go.mod
View File

@ -100,6 +100,7 @@ require (
github.com/JohannesKaufmann/dom v0.2.0 // indirect github.com/JohannesKaufmann/dom v0.2.0 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Tencent/WeKnora/pluginsdk v0.0.0-00010101000000-000000000000
github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/apache/arrow-go/v18 v18.5.1 // indirect github.com/apache/arrow-go/v18 v18.5.1 // indirect
@ -359,3 +360,5 @@ replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelg
// The anydoc Go bindings are not published yet (firecrawl/anydoc#30 is still // The anydoc Go bindings are not published yet (firecrawl/anydoc#30 is still
// open), so they are vendored. Drop this replace once upstream tags go/vX.Y.Z. // open), so they are vendored. Drop this replace once upstream tags go/vX.Y.Z.
replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go
replace github.com/Tencent/WeKnora/pluginsdk => ./pluginsdk

View File

@ -658,7 +658,9 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
} }
ds.ErrorMessage = syncLog.ErrorMessage ds.ErrorMessage = syncLog.ErrorMessage
_ = s.dsRepo.Update(ctx, ds) _ = s.dsRepo.Update(ctx, ds)
return err // Retrying cannot bring a connector back (its plugin was removed or
// disabled); the next scheduled or manual sync will try again.
return fmt.Errorf("%w: %w", asynq.SkipRetry, err)
} }
// Parse configuration // Parse configuration
@ -1247,7 +1249,9 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {
} }
// ValidateCredentials tests connectivity using raw credentials without persisting anything. // ValidateCredentials tests connectivity using raw credentials without persisting anything.
func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error { func (s *DataSourceService) ValidateCredentials(
ctx context.Context, connectorType string, credentials, settings map[string]interface{},
) error {
connector, err := s.connectorRegistry.Get(connectorType) connector, err := s.connectorRegistry.Get(connectorType)
if err != nil { if err != nil {
return err return err
@ -1255,6 +1259,7 @@ func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorTy
config := &types.DataSourceConfig{ config := &types.DataSourceConfig{
Type: connectorType, Type: connectorType,
Credentials: credentials, Credentials: credentials,
Settings: settings,
} }
if err := connector.Validate(ctx, config); err != nil { if err := connector.Validate(ctx, config); err != nil {
return err return err

View File

@ -14,11 +14,46 @@ import (
// webSearchProviderService implements interfaces.WebSearchProviderService // webSearchProviderService implements interfaces.WebSearchProviderService
type webSearchProviderService struct { type webSearchProviderService struct {
repo interfaces.WebSearchProviderRepository repo interfaces.WebSearchProviderRepository
// registry knows the provider types installed plugins contribute.
registry *infra_web_search.Registry
} }
// NewWebSearchProviderService creates a new web search provider service // NewWebSearchProviderService creates a new web search provider service
func NewWebSearchProviderService(repo interfaces.WebSearchProviderRepository) interfaces.WebSearchProviderService { func NewWebSearchProviderService(
return &webSearchProviderService{repo: repo} repo interfaces.WebSearchProviderRepository, registry *infra_web_search.Registry,
) interfaces.WebSearchProviderService {
return &webSearchProviderService{repo: repo, registry: registry}
}
// validType accepts builtin types and the types of loaded plugins.
func (s *webSearchProviderService) validType(t types.WebSearchProviderType) bool {
if isValidProviderType(t) {
return true
}
if s.registry == nil {
return false
}
_, ok := s.registry.PluginType(string(t))
return ok
}
// validateParams checks builtin parameters by type and plugin parameters
// against the plugin's instance schema.
func (s *webSearchProviderService) validateParams(
t types.WebSearchProviderType, params types.WebSearchProviderParameters,
) error {
if s.registry != nil {
if pt, ok := s.registry.PluginType(string(t)); ok {
if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
return err
}
if pt.Validate != nil {
return pt.Validate(params)
}
return nil
}
}
return validateProviderParameters(t, params)
} }
// CreateProvider creates a new web search provider configuration. // CreateProvider creates a new web search provider configuration.
@ -27,11 +62,11 @@ func (s *webSearchProviderService) CreateProvider(ctx context.Context, provider
return fmt.Errorf("tenant ID is required") return fmt.Errorf("tenant ID is required")
} }
if !isValidProviderType(provider.Provider) { if !s.validType(provider.Provider) {
return fmt.Errorf("invalid provider type: %s", provider.Provider) return fmt.Errorf("invalid provider type: %s", provider.Provider)
} }
if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil { if err := s.validateParams(provider.Provider, provider.Parameters); err != nil {
return err return err
} }
@ -52,7 +87,7 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
} }
// Validate provider type if set // Validate provider type if set
if provider.Provider != "" && !isValidProviderType(provider.Provider) { if provider.Provider != "" && !s.validType(provider.Provider) {
return fmt.Errorf("invalid provider type: %s", provider.Provider) return fmt.Errorf("invalid provider type: %s", provider.Provider)
} }
@ -63,7 +98,7 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
} }
if provider.Provider != "" { if provider.Provider != "" {
if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil { if err := s.validateParams(provider.Provider, provider.Parameters); err != nil {
return err return err
} }
} }

View File

@ -74,6 +74,8 @@ import (
"github.com/Tencent/WeKnora/internal/plugin/activate" "github.com/Tencent/WeKnora/internal/plugin/activate"
pluginbuiltin "github.com/Tencent/WeKnora/internal/plugin/builtin" pluginbuiltin "github.com/Tencent/WeKnora/internal/plugin/builtin"
plugindriver "github.com/Tencent/WeKnora/internal/plugin/driver" plugindriver "github.com/Tencent/WeKnora/internal/plugin/driver"
pluginhost "github.com/Tencent/WeKnora/internal/plugin/host"
pluginmanifest "github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile" "github.com/Tencent/WeKnora/internal/plugin/reconcile"
pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry" pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry"
plugintenancy "github.com/Tencent/WeKnora/internal/plugin/tenancy" plugintenancy "github.com/Tencent/WeKnora/internal/plugin/tenancy"
@ -165,6 +167,10 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(activate.NewMCPServers)) must(container.Provide(activate.NewMCPServers))
must(container.Provide(activate.NewSkills)) must(container.Provide(activate.NewSkills))
must(container.Provide(activate.NewModelVendors)) must(container.Provide(activate.NewModelVendors))
must(container.Provide(pluginhost.NewManager))
must(container.Provide(newPluginInvoker))
must(container.Provide(activate.NewWebSearch))
must(container.Provide(activate.NewConnectors))
must(container.Provide(newMCPServiceRepository)) must(container.Provide(newMCPServiceRepository))
must(container.Provide(repository.NewMCPToolApprovalRepository)) must(container.Provide(repository.NewMCPToolApprovalRepository))
must(container.Provide(repository.NewMCPOAuthRepository)) must(container.Provide(repository.NewMCPOAuthRepository))
@ -1835,7 +1841,8 @@ func newPluginRegistry(
// declarative packages today; host, remote and kubernetes drivers join this // declarative packages today; host, remote and kubernetes drivers join this
// set. // set.
func newPluginDrivers(reg *pluginregistry.Registry, r *reconcile.Reconciler) *plugindriver.Set { func newPluginDrivers(reg *pluginregistry.Registry, r *reconcile.Reconciler) *plugindriver.Set {
return plugindriver.NewSet(plugindriver.NewBuiltin(reg.Plugin), r.Driver()) return plugindriver.NewSet(plugindriver.NewBuiltin(reg.Plugin),
r.Driver(pluginmanifest.RuntimeDeclarative), r.Driver(pluginmanifest.RuntimeHost))
} }
// installPluginGate gives the integration handlers the tenant plugin switches, // installPluginGate gives the integration handlers the tenant plugin switches,

View File

@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/handler" "github.com/Tencent/WeKnora/internal/handler"
"github.com/Tencent/WeKnora/internal/plugin/activate" "github.com/Tencent/WeKnora/internal/plugin/activate"
"github.com/Tencent/WeKnora/internal/plugin/host"
"github.com/Tencent/WeKnora/internal/plugin/install" "github.com/Tencent/WeKnora/internal/plugin/install"
"github.com/Tencent/WeKnora/internal/plugin/reconcile" "github.com/Tencent/WeKnora/internal/plugin/reconcile"
pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry" pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry"
@ -34,13 +35,19 @@ func newPluginPackageStore(cfg *config.Config) (reconcile.PackageStore, error) {
type pluginActivators struct { type pluginActivators struct {
dig.In dig.In
MCP *activate.MCPServers Host *host.Manager
Skills *activate.Skills WebSearch *activate.WebSearch
Vendors *activate.ModelVendors Connectors *activate.Connectors
MCP *activate.MCPServers
Skills *activate.Skills
Vendors *activate.ModelVendors
Invoker *activate.Invoker
} }
// list orders the activators: the host first, so a code plugin's process is
// running before anything routes calls to it.
func (a pluginActivators) list() []reconcile.Activator { func (a pluginActivators) list() []reconcile.Activator {
return []reconcile.Activator{a.Vendors, a.MCP, a.Skills} return []reconcile.Activator{a.Host, a.WebSearch, a.Connectors, a.Vendors, a.MCP, a.Skills}
} }
// bindPluginActivators hands the activators what they need once the plugin // bindPluginActivators hands the activators what they need once the plugin
@ -50,10 +57,14 @@ func bindPluginActivators(
skills *service.TenantSkillService, skills *service.TenantSkillService,
) { ) {
a.MCP.Bind(t, repo) a.MCP.Bind(t, repo)
a.Invoker.Bind(t, repo)
a.Skills.Bind(t) a.Skills.Bind(t)
skills.SetPluginSkills(a.Skills) skills.SetPluginSkills(a.Skills)
} }
// newPluginInvoker reaches code plugins through this node's plugin host.
func newPluginInvoker(h *host.Manager) *activate.Invoker { return activate.NewInvoker(h) }
func newMCPServiceRepository(db *gorm.DB, plugins *activate.MCPServers) interfaces.MCPServiceRepository { func newMCPServiceRepository(db *gorm.DB, plugins *activate.MCPServers) interfaces.MCPServiceRepository {
return plugins.Repository(repository.NewMCPServiceRepository(db)) return plugins.Repository(repository.NewMCPServiceRepository(db))
} }
@ -80,8 +91,13 @@ func newPluginInstaller(
// startPluginReconciler loads installed plugins before the server takes // startPluginReconciler loads installed plugins before the server takes
// traffic, then keeps this node in step with the others. // traffic, then keeps this node in step with the others.
func startPluginReconciler(r *reconcile.Reconciler, cleaner interfaces.ResourceCleaner) { func startPluginReconciler(r *reconcile.Reconciler, hostManager *host.Manager, cleaner interfaces.ResourceCleaner) {
hostManager.SetReporter(r)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
r.Start(ctx) r.Start(ctx)
cleaner.RegisterWithName("PluginReconciler", func() error { cancel(); return nil }) cleaner.RegisterWithName("PluginReconciler", func() error {
cancel()
hostManager.Close()
return nil
})
} }

View File

@ -2,8 +2,10 @@ package datasource
import ( import (
"context" "context"
"fmt"
"math" "math"
"sort" "sort"
"sync"
"github.com/Tencent/WeKnora/internal/plugin/configschema" "github.com/Tencent/WeKnora/internal/plugin/configschema"
"github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types"
@ -121,15 +123,21 @@ type FullSyncWithCursor interface {
) ([]types.FetchedItem, *types.SyncCursor, error) ) ([]types.FetchedItem, *types.SyncCursor, error)
} }
// ConnectorRegistry manages the registration and lookup of available connectors // ConnectorRegistry manages the registration and lookup of available
// connectors. Installed plugins add and remove connectors while the server
// runs, so it is safe for concurrent use.
type ConnectorRegistry struct { type ConnectorRegistry struct {
mu sync.RWMutex
connectors map[string]Connector connectors map[string]Connector
// plugins holds the metadata of connectors installed plugins registered.
plugins map[string]ConnectorMetadata
} }
// NewConnectorRegistry creates a new connector registry // NewConnectorRegistry creates a new connector registry
func NewConnectorRegistry() *ConnectorRegistry { func NewConnectorRegistry() *ConnectorRegistry {
return &ConnectorRegistry{ return &ConnectorRegistry{
connectors: make(map[string]Connector), connectors: make(map[string]Connector),
plugins: make(map[string]ConnectorMetadata),
} }
} }
@ -141,12 +149,51 @@ func (r *ConnectorRegistry) Register(connector Connector) error {
if connector.Type() == "" { if connector.Type() == "" {
return ErrConnectorTypeEmpty return ErrConnectorTypeEmpty
} }
r.mu.Lock()
defer r.mu.Unlock()
r.connectors[connector.Type()] = connector r.connectors[connector.Type()] = connector
return nil return nil
} }
// RegisterPlugin adds or replaces a plugin's connector with the metadata it
// is listed under. It refuses to shadow a builtin connector.
func (r *ConnectorRegistry) RegisterPlugin(connector Connector, meta ConnectorMetadata) error {
if connector == nil {
return ErrConnectorNil
}
t := connector.Type()
if t == "" {
return ErrConnectorTypeEmpty
}
r.mu.Lock()
defer r.mu.Unlock()
if _, exists := r.connectors[t]; exists {
if _, isPlugin := r.plugins[t]; !isPlugin {
return fmt.Errorf("connector type %s already exists", t)
}
}
meta.Type = t
r.connectors[t] = connector
r.plugins[t] = meta
return nil
}
// Unregister removes a plugin's connector; builtins stay. Syncs already
// running keep the connector they started with.
func (r *ConnectorRegistry) Unregister(connectorType string) {
r.mu.Lock()
defer r.mu.Unlock()
if _, isPlugin := r.plugins[connectorType]; !isPlugin {
return
}
delete(r.plugins, connectorType)
delete(r.connectors, connectorType)
}
// Get retrieves a connector by type // Get retrieves a connector by type
func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) { func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
r.mu.RLock()
defer r.mu.RUnlock()
connector, exists := r.connectors[connectorType] connector, exists := r.connectors[connectorType]
if !exists { if !exists {
return nil, ErrConnectorNotFound return nil, ErrConnectorNotFound
@ -156,6 +203,8 @@ func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
// List returns all registered connector types // List returns all registered connector types
func (r *ConnectorRegistry) List() []string { func (r *ConnectorRegistry) List() []string {
r.mu.RLock()
defer r.mu.RUnlock()
types := make([]string, 0, len(r.connectors)) types := make([]string, 0, len(r.connectors))
for t := range r.connectors { for t := range r.connectors {
types = append(types, t) types = append(types, t)
@ -182,6 +231,16 @@ type ConnectorMetadata struct {
// ConfigSchema describes DataSourceConfig.Credentials for this // ConfigSchema describes DataSourceConfig.Credentials for this
// connector; the editor renders the credential form from it. // connector; the editor renders the credential form from it.
ConfigSchema *configschema.Schema `json:"config_schema,omitempty"` ConfigSchema *configschema.Schema `json:"config_schema,omitempty"`
// SettingsSchema describes DataSourceConfig.Settings (non-secret
// options) for connectors whose settings form is not built into the
// frontend, such as plugin connectors.
SettingsSchema *configschema.Schema `json:"settings_schema,omitempty"`
// Names and Descriptions localize Name and Description, keyed by
// locale, for plugin connectors (builtins use frontend locale keys).
Names map[string]string `json:"names,omitempty"`
Descriptions map[string]string `json:"descriptions,omitempty"`
// PluginID names the installed plugin providing the connector.
PluginID string `json:"plugin_id,omitempty"`
} }
// GetConnectorMetadata returns metadata for all available connectors // GetConnectorMetadata returns metadata for all available connectors
@ -346,6 +405,8 @@ func ListAvailableConnectors() []ConnectorMetadata {
// ListAvailableConnectors order. A registered connector without an entry in // ListAvailableConnectors order. A registered connector without an entry in
// ConnectorMetadataRegistry is listed last under its type. // ConnectorMetadataRegistry is listed last under its type.
func (r *ConnectorRegistry) Metadata() []ConnectorMetadata { func (r *ConnectorRegistry) Metadata() []ConnectorMetadata {
r.mu.RLock()
defer r.mu.RUnlock()
out := make([]ConnectorMetadata, 0, len(r.connectors)) out := make([]ConnectorMetadata, 0, len(r.connectors))
seen := make(map[string]bool, len(r.connectors)) seen := make(map[string]bool, len(r.connectors))
for _, meta := range ListAvailableConnectors() { for _, meta := range ListAvailableConnectors() {
@ -362,6 +423,10 @@ func (r *ConnectorRegistry) Metadata() []ConnectorMetadata {
} }
sort.Strings(rest) sort.Strings(rest)
for _, t := range rest { for _, t := range rest {
if meta, ok := r.plugins[t]; ok {
out = append(out, meta)
continue
}
out = append(out, ConnectorMetadata{Type: t, Name: t, Priority: math.MaxInt32}) out = append(out, ConnectorMetadata{Type: t, Name: t, Priority: math.MaxInt32})
} }
return out return out

View File

@ -326,13 +326,18 @@ func (h *DataSourceHandler) ValidateCredentials(c *gin.Context) {
var req struct { var req struct {
Type string `json:"type" binding:"required"` Type string `json:"type" binding:"required"`
Credentials map[string]interface{} `json:"credentials" binding:"required"` Credentials map[string]interface{} `json:"credentials" binding:"required"`
Settings map[string]interface{} `json:"settings"`
} }
if err := c.ShouldBindJSON(&req); err != nil { if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: type and credentials are required"}) c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: type and credentials are required"})
return return
} }
if !h.pluginFilter(c)(manifest.PointConnectors, req.Type) {
c.JSON(http.StatusBadRequest, gin.H{"error": disabledIntegrationError})
return
}
if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials); err != nil { if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials, req.Settings); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return return
} }

View File

@ -329,6 +329,9 @@ func (h *WebSearchProviderHandler) DeleteProvider(c *gin.Context) {
func (h *WebSearchProviderHandler) ListProviderTypes(c *gin.Context) { func (h *WebSearchProviderHandler) ListProviderTypes(c *gin.Context) {
enabled := h.pluginFilter(c) enabled := h.pluginFilter(c)
all := types.GetWebSearchProviderTypes() all := types.GetWebSearchProviderTypes()
if h.registry != nil {
all = append(all, h.registry.PluginTypes()...)
}
out := make([]types.WebSearchProviderTypeInfo, 0, len(all)) out := make([]types.WebSearchProviderTypeInfo, 0, len(all))
for _, info := range all { for _, info := range all {
if enabled(manifest.PointWebSearch, info.ID) { if enabled(manifest.PointWebSearch, info.ID) {
@ -408,6 +411,11 @@ func (h *WebSearchProviderHandler) TestProviderRaw(c *gin.Context) {
return return
} }
if !h.pluginFilter(c)(manifest.PointWebSearch, req.Provider) {
_ = c.Error(errors.NewBadRequestError(disabledIntegrationError))
return
}
if err := h.doTestSearch(ctx, req.Provider, req.Parameters); err != nil { if err := h.doTestSearch(ctx, req.Provider, req.Parameters); err != nil {
logger.Warnf(ctx, "Web search provider test failed: %v", err) logger.Warnf(ctx, "Web search provider test failed: %v", err)
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()}) c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})

View File

@ -17,16 +17,74 @@ type ProviderFactory func(params types.WebSearchProviderParameters) (interfaces.
// Instances are created on-demand with tenant-specific parameters. // Instances are created on-demand with tenant-specific parameters.
type Registry struct { type Registry struct {
factories map[string]ProviderFactory factories map[string]ProviderFactory
mu sync.RWMutex // plugins describes the types installed plugins registered; builtin
// types are described by types.GetWebSearchProviderTypes.
plugins map[string]PluginType
mu sync.RWMutex
}
// PluginType is a provider type an installed plugin contributes.
type PluginType struct {
Info types.WebSearchProviderTypeInfo
// Validate checks instance parameters against the plugin's schema.
Validate func(params types.WebSearchProviderParameters) error
} }
// NewRegistry creates a new web search provider registry // NewRegistry creates a new web search provider registry
func NewRegistry() *Registry { func NewRegistry() *Registry {
return &Registry{ return &Registry{
factories: make(map[string]ProviderFactory), factories: make(map[string]ProviderFactory),
plugins: make(map[string]PluginType),
} }
} }
// RegisterPlugin adds or replaces a plugin's provider type. It refuses to
// shadow a builtin type.
func (r *Registry) RegisterPlugin(t PluginType, factory ProviderFactory) error {
r.mu.Lock()
defer r.mu.Unlock()
id := t.Info.ID
if _, exists := r.factories[id]; exists {
if _, isPlugin := r.plugins[id]; !isPlugin {
return fmt.Errorf("web search provider type %s already exists", id)
}
}
r.factories[id] = factory
r.plugins[id] = t
return nil
}
// Unregister removes a plugin's provider type; builtins stay.
func (r *Registry) Unregister(id string) {
r.mu.Lock()
defer r.mu.Unlock()
if _, isPlugin := r.plugins[id]; !isPlugin {
return
}
delete(r.plugins, id)
delete(r.factories, id)
}
// PluginType returns a plugin provider type.
func (r *Registry) PluginType(id string) (PluginType, bool) {
r.mu.RLock()
defer r.mu.RUnlock()
t, ok := r.plugins[id]
return t, ok
}
// PluginTypes returns the plugin provider types, sorted by ID.
func (r *Registry) PluginTypes() []types.WebSearchProviderTypeInfo {
r.mu.RLock()
defer r.mu.RUnlock()
out := make([]types.WebSearchProviderTypeInfo, 0, len(r.plugins))
for _, t := range r.plugins {
out = append(out, t.Info)
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out
}
// Register registers a provider type factory by ID // Register registers a provider type factory by ID
func (r *Registry) Register(id string, factory ProviderFactory) { func (r *Registry) Register(id string, factory ProviderFactory) {
r.mu.Lock() r.mu.Lock()

View File

@ -0,0 +1,354 @@
package activate
import (
"context"
"encoding/json"
"fmt"
"sort"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/plugin/configschema"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Connectors registers code plugins' data source connectors. A plugin
// connector's instanceSchema splits in two: fields in x-group "settings"
// are non-secret options (DataSourceConfig.Settings), everything else is
// credentials (encrypted at rest).
type Connectors struct {
iv *Invoker
registry *datasource.ConnectorRegistry
mu sync.Mutex
registered map[string][]string
}
// NewConnectors creates the connector activator.
func NewConnectors(iv *Invoker, registry *datasource.ConnectorRegistry) *Connectors {
return &Connectors{iv: iv, registry: registry, registered: map[string][]string{}}
}
// Name implements reconcile.Activator.
func (a *Connectors) Name() string { return "connectors" }
// settingsGroup marks instanceSchema fields that are settings, not
// credentials.
const settingsGroup = "settings"
// Activate implements reconcile.Activator: all of a plugin's connectors or
// none.
func (a *Connectors) Activate(_ context.Context, l *reconcile.Loaded) error {
var ids []string
for _, c := range l.Manifest.Contributes[manifest.PointConnectors] {
id := manifest.QualifiedID(l.Manifest.ID, c.ID)
meta, err := connectorMetadata(l, c, id)
if err == nil {
rc := &remoteConnector{iv: a.iv, m: l.Manifest, local: c.ID, typeID: id}
err = a.registry.RegisterPlugin(rc, meta)
}
if err != nil {
for _, done := range ids {
a.registry.Unregister(done)
}
return fmt.Errorf("connector %s: %w", c.ID, err)
}
ids = append(ids, id)
}
a.mu.Lock()
a.registered[l.Manifest.ID] = ids
a.mu.Unlock()
return nil
}
// Deactivate implements reconcile.Activator.
func (a *Connectors) Deactivate(_ context.Context, pluginID string) error {
a.mu.Lock()
ids := a.registered[pluginID]
delete(a.registered, pluginID)
a.mu.Unlock()
for _, id := range ids {
a.registry.Unregister(id)
}
return nil
}
func connectorMetadata(
l *reconcile.Loaded,
c manifest.Contribution,
id string,
) (datasource.ConnectorMetadata, error) {
src, err := instanceSchema(l.Package, c)
if err != nil {
return datasource.ConnectorMetadata{}, err
}
creds, settings := splitConnectorSchema(src)
icon, err := iconDataURI(l.Package, c.Icon)
if err != nil {
return datasource.ConnectorMetadata{}, err
}
caps := c.Capabilities
if caps == nil {
caps = []string{}
}
return datasource.ConnectorMetadata{
Type: id, Name: c.Name.Default, Names: c.Name.Locales,
Description: c.Description.Default, Descriptions: c.Description.Locales,
Icon: icon, Priority: 1000 + c.Order, AuthType: "plugin", Capabilities: caps,
ConfigSchema: creds, SettingsSchema: settings, PluginID: l.Manifest.ID,
}, nil
}
// splitConnectorSchema separates settings fields from credential fields.
func splitConnectorSchema(src *configschema.Schema) (creds, settings *configschema.Schema) {
if src == nil {
return nil, nil
}
required := map[string]bool{}
for _, k := range src.Required {
required[k] = true
}
creds, settings = configschema.Object(), configschema.Object()
keys := make([]string, 0, len(src.Properties))
for k := range src.Properties {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
prop := *src.Properties[k]
if prop.Group == settingsGroup && !prop.Secret {
prop.Group = ""
settings.Set(k, &prop, required[k])
continue
}
creds.Set(k, &prop, required[k])
}
if len(creds.Properties) == 0 {
creds = nil
}
if len(settings.Properties) == 0 {
settings = nil
}
return creds, settings
}
// remoteConnector is a plugin connector. It streams, so the sync service
// interleaves fetch, ingest and checkpoint and resumes after a timeout.
type remoteConnector struct {
iv *Invoker
m *manifest.Manifest
local string
typeID string
}
var _ datasource.FullStreamingConnector = (*remoteConnector)(nil)
func (r *remoteConnector) Type() string { return r.typeID }
func instanceOf(cfg *types.DataSourceConfig) map[string]any {
if cfg == nil {
return map[string]any{}
}
return map[string]any{
"credentials": nonNil(cfg.Credentials),
"settings": nonNil(cfg.Settings),
"resourceIds": cfg.ResourceIDs,
}
}
func nonNil(m map[string]any) map[string]any {
if m == nil {
return map[string]any{}
}
return m
}
// Validate implements datasource.Connector.
func (r *remoteConnector) Validate(ctx context.Context, cfg *types.DataSourceConfig) error {
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
defer cancel()
return r.iv.Call(ctx, r.m, pluginapi.ConnectorValidatePath(r.local), instanceOf(cfg), nil, nil)
}
// ListResources implements datasource.Connector.
func (r *remoteConnector) ListResources(
ctx context.Context, cfg *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
defer cancel()
var out pluginapi.ListResourcesOutput
err := r.iv.Call(ctx, r.m, pluginapi.ConnectorListResourcesPath(r.local), instanceOf(cfg),
pluginapi.ListResourcesInput{ParentID: parentID}, &out)
if err != nil {
return nil, err
}
res := make([]types.Resource, 0, len(out.Resources))
for _, x := range out.Resources {
item := types.Resource{
ExternalID: x.ExternalID, Name: x.Name, Type: x.Type, Description: x.Description, URL: x.URL,
ParentID: x.ParentID, HasChildren: x.HasChildren, Metadata: x.Metadata,
}
if x.ModifiedAt != nil {
item.ModifiedAt = *x.ModifiedAt
}
res = append(res, item)
}
return res, nil
}
// ResolveResourceAncestors implements datasource.Connector.
func (r *remoteConnector) ResolveResourceAncestors(
ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
) ([]string, error) {
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
defer cancel()
var out pluginapi.ResolveAncestorsOutput
err := r.iv.Call(ctx, r.m, pluginapi.ConnectorResolveAncestorsPath(r.local), instanceOf(cfg),
pluginapi.ResolveAncestorsInput{ResourceIDs: resourceIDs}, &out)
return out.Ancestors, err
}
// collector gathers a stream for the non-streaming entry points.
type collector struct {
items []types.FetchedItem
cursor *types.SyncCursor
}
func (c *collector) Emit(_ context.Context, item types.FetchedItem) error {
c.items = append(c.items, item)
return nil
}
func (c *collector) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
c.cursor = cursor
return nil
}
// FetchAll implements datasource.Connector.
func (r *remoteConnector) FetchAll(
ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
) ([]types.FetchedItem, error) {
c := &collector{}
scoped := *cfg
if len(resourceIDs) > 0 {
scoped.ResourceIDs = resourceIDs
}
_, err := r.fetch(ctx, &scoped, pluginapi.FetchFull, nil, c)
return c.items, err
}
// FetchIncremental implements datasource.Connector.
func (r *remoteConnector) FetchIncremental(
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor,
) ([]types.FetchedItem, *types.SyncCursor, error) {
c := &collector{}
next, err := r.fetch(ctx, cfg, pluginapi.FetchIncremental, cursor, c)
return c.items, next, err
}
// FetchStream implements datasource.StreamingConnector: incremental from a
// cursor, full without one.
func (r *remoteConnector) FetchStream(
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
) (*types.SyncCursor, error) {
mode := pluginapi.FetchIncremental
if cursor == nil {
mode = pluginapi.FetchFull
}
return r.fetch(ctx, cfg, mode, cursor, h)
}
// FetchFullStream implements datasource.FullStreamingConnector: every item
// again, with the previous cursor for deletion reconciliation.
func (r *remoteConnector) FetchFullStream(
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
) (*types.SyncCursor, error) {
return r.fetch(ctx, cfg, pluginapi.FetchFull, cursor, h)
}
func (r *remoteConnector) fetch(
ctx context.Context, cfg *types.DataSourceConfig, mode string, cursor *types.SyncCursor,
h datasource.StreamHandler,
) (*types.SyncCursor, error) {
in := pluginapi.FetchInput{Mode: mode, Cursor: toPluginCursor(cursor)}
if cfg != nil {
in.ResourceIDs = cfg.ResourceIDs
}
end, err := r.iv.Stream(ctx, r.m, pluginapi.ConnectorFetchPath(r.local), instanceOf(cfg), in,
func(ev pluginapi.Event) error {
switch ev.Type {
case pluginapi.EventItem:
var it pluginapi.FetchedItem
if err := json.Unmarshal(ev.Data, &it); err != nil {
return fmt.Errorf("decode fetched item: %w", err)
}
return h.Emit(ctx, fromPluginItem(it))
case pluginapi.EventCheckpoint:
var c pluginapi.Cursor
if err := json.Unmarshal(ev.Data, &c); err != nil {
return fmt.Errorf("decode checkpoint: %w", err)
}
return h.Checkpoint(ctx, fromPluginCursor(&c))
}
return nil
})
if err != nil {
return nil, err
}
var final pluginapi.Cursor
if len(end) > 0 {
if err := json.Unmarshal(end, &final); err != nil {
return nil, fmt.Errorf("decode final cursor: %w", err)
}
}
next := fromPluginCursor(&final)
if next.LastSyncTime.IsZero() {
next.LastSyncTime = time.Now()
}
return next, nil
}
func toPluginCursor(c *types.SyncCursor) *pluginapi.Cursor {
if c == nil {
return nil
}
out := &pluginapi.Cursor{State: c.ConnectorCursor}
if !c.LastSyncTime.IsZero() {
t := c.LastSyncTime
out.LastSyncTime = &t
}
return out
}
func fromPluginCursor(c *pluginapi.Cursor) *types.SyncCursor {
out := &types.SyncCursor{ConnectorCursor: c.State}
if c.LastSyncTime != nil {
out.LastSyncTime = *c.LastSyncTime
}
if out.ConnectorCursor == nil {
out.ConnectorCursor = map[string]any{}
}
return out
}
func fromPluginItem(it pluginapi.FetchedItem) types.FetchedItem {
out := types.FetchedItem{
ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType,
FileName: it.FileName, URL: it.URL, Metadata: it.Metadata, IsDeleted: it.IsDeleted,
SourceResourceID: it.SourceResourceID,
}
if it.UpdatedAt != nil {
out.UpdatedAt = *it.UpdatedAt
}
if it.CreatedAt != nil {
out.CreatedAt = *it.CreatedAt
}
if out.ContentType == "" {
out.ContentType = "text/markdown"
}
return out
}

View File

@ -0,0 +1,202 @@
package activate
import (
"context"
"errors"
"fmt"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/plugin/pkg"
"github.com/Tencent/WeKnora/internal/plugin/plugintest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
const feedPluginManifest = `schemaVersion: 1
id: acme.feeds
version: 1.0.0
apiVersion: weknora.plugin/v1
name: { en-US: ACME Feeds }
publisher: { id: acme }
runtime: { type: host, kind: binary, entry: bin/feeds }
contributes:
connectors:
- id: feed
name: { en-US: ACME Feed, zh-CN: ACME 订阅 }
capabilities: [incremental]
instanceSchema: schemas/feed.yaml
`
const feedSchema = `type: object
required: [feed_urls]
properties:
token: { type: string, x-secret: true }
feed_urls: { type: string, x-widget: textarea, x-group: settings }
`
// feedPlugin serves numbered items and resumes from the cursor.
type feedPlugin struct{}
func (feedPlugin) Validate(_ context.Context, _ *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
if cfg.Settings["feed_urls"] == nil {
return pluginapi.InvalidConfig("feed_urls is required", map[string]string{"settings.feed_urls": "required"})
}
if cfg.Credentials["token"] != "t-1" {
return pluginapi.Errorf(pluginapi.CodeUnauthorized, "bad token")
}
return nil
}
func (feedPlugin) ListResources(
_ context.Context,
_ *pluginsdk.Call,
cfg pluginsdk.ConnectorConfig,
parent string,
) ([]pluginapi.Resource, error) {
if parent != "" {
return nil, nil
}
var out []pluginapi.Resource
for _, u := range strings.Split(fmt.Sprint(cfg.Settings["feed_urls"]), "\n") {
out = append(out, pluginapi.Resource{ExternalID: u, Name: u, Type: "feed"})
}
return out, nil
}
func (feedPlugin) Fetch(
_ context.Context,
_ *pluginsdk.Call,
_ pluginsdk.ConnectorConfig,
in pluginapi.FetchInput,
s *pluginsdk.Stream,
) (*pluginapi.Cursor, error) {
start := 0
if in.Mode == pluginapi.FetchIncremental && in.Cursor != nil {
start = int(in.Cursor.State["seen"].(float64))
}
for i := start; i < 3; i++ {
item := pluginapi.FetchedItem{
ExternalID: fmt.Sprint("post-", i), Title: fmt.Sprint("Post ", i), Content: []byte("# hi"),
}
if err := s.Item(item); err != nil {
return nil, err
}
if err := s.Checkpoint(pluginapi.Cursor{State: map[string]any{"seen": i + 1}}); err != nil {
return nil, err
}
}
return &pluginapi.Cursor{State: map[string]any{"seen": 3}}, nil
}
type recordingHandler struct {
items []string
checkpoints int
}
func (h *recordingHandler) Emit(_ context.Context, item types.FetchedItem) error {
h.items = append(h.items, item.ExternalID+"|"+item.ContentType)
return nil
}
func (h *recordingHandler) Checkpoint(context.Context, *types.SyncCursor) error {
h.checkpoints++
return nil
}
func TestPluginConnector(t *testing.T) {
ctx := context.Background()
plugin := pluginsdk.New(pluginsdk.Info{ID: "acme.feeds", Version: "1.0.0"})
plugin.Connector("feed", feedPlugin{})
srv := httptest.NewServer(plugin.Handler())
defer srv.Close()
p, err := pkg.Open(plugintest.Zip(t, map[string]string{
"plugin.yaml": feedPluginManifest, "bin/feeds": "x", "schemas/feed.yaml": feedSchema,
}))
if err != nil {
t.Fatal(err)
}
registry := datasource.NewConnectorRegistry()
a := NewConnectors(NewInvoker(fakeClients{client.New(srv.URL, nil, nil)}), registry)
if err := a.Activate(ctx, &reconcile.Loaded{Manifest: p.Manifest, Package: p}); err != nil {
t.Fatalf("Activate: %v", err)
}
var meta datasource.ConnectorMetadata
for _, m := range registry.Metadata() {
if m.Type == "acme.feeds/feed" {
meta = m
}
}
if meta.Names["zh-CN"] != "ACME 订阅" || meta.PluginID != "acme.feeds" ||
meta.ConfigSchema.Properties["token"] == nil || meta.SettingsSchema.Properties["feed_urls"] == nil ||
meta.ConfigSchema.Properties["feed_urls"] != nil {
t.Fatalf("metadata = %+v", meta)
}
conn, err := registry.Get("acme.feeds/feed")
if err != nil {
t.Fatal(err)
}
cfg := &types.DataSourceConfig{
Credentials: map[string]any{"token": "t-1"},
Settings: map[string]any{"feed_urls": "https://a.example/rss\nhttps://b.example/rss"},
}
err = conn.Validate(ctx, &types.DataSourceConfig{Credentials: map[string]any{"token": "t-1"}})
var pe *pluginapi.Error
if err == nil || err.Error() != "feed_urls is required" ||
!errors.As(err, &pe) || pe.Code != pluginapi.CodeInvalidConfig {
t.Fatalf("Validate without settings = %v; want the plugin's message with its code reachable", err)
}
if err := conn.Validate(ctx, cfg); err != nil {
t.Fatalf("Validate: %v", err)
}
res, err := conn.ListResources(ctx, cfg, "")
if err != nil || len(res) != 2 || res[1].ExternalID != "https://b.example/rss" {
t.Fatalf("ListResources = %+v, %v", res, err)
}
streaming := conn.(datasource.StreamingConnector)
h := &recordingHandler{}
cursor, err := streaming.FetchStream(ctx, cfg, nil, h)
if err != nil {
t.Fatal(err)
}
if strings.Join(h.items, ",") != "post-0|text/markdown,post-1|text/markdown,post-2|text/markdown" ||
h.checkpoints != 3 || cursor.ConnectorCursor["seen"].(float64) != 3 || cursor.LastSyncTime.IsZero() {
t.Fatalf("items=%v checkpoints=%d cursor=%+v", h.items, h.checkpoints, cursor)
}
cursor.ConnectorCursor["seen"] = float64(2)
items, next, err := conn.FetchIncremental(ctx, cfg, cursor)
if err != nil || len(items) != 1 || items[0].ExternalID != "post-2" ||
next.ConnectorCursor["seen"].(float64) != 3 {
t.Fatalf("incremental = %+v %+v %v", items, next, err)
}
if err := a.Deactivate(ctx, "acme.feeds"); err != nil {
t.Fatal(err)
}
if _, err := registry.Get("acme.feeds/feed"); err == nil {
t.Fatal("connector survived Deactivate")
}
}
func TestPluginConnectorCannotShadowABuiltin(t *testing.T) {
registry := datasource.NewConnectorRegistry()
if err := registry.Register(&remoteConnector{typeID: "rss"}); err != nil {
t.Fatal(err)
}
if err := registry.RegisterPlugin(&remoteConnector{typeID: "rss"}, datasource.ConnectorMetadata{}); err == nil {
t.Fatal("a plugin must not replace a builtin connector")
}
registry.Unregister("rss")
if _, err := registry.Get("rss"); err != nil {
t.Fatal("Unregister must not remove builtins")
}
}

View File

@ -0,0 +1,153 @@
package activate
import (
"context"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/plugin/install"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/tenancy"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// ClientSource reaches running code plugins (the node's plugin host).
type ClientSource interface {
Client(pluginID string) (*client.Client, error)
}
// Invoker is how the remote adapters call a code plugin: it finds the
// process and builds the protocol envelope from the request context and the
// plugin's decrypted system and tenant configuration.
type Invoker struct {
clients ClientSource
mu sync.RWMutex
tenancy *tenancy.Service
plugins interfaces.PluginRepository
}
// NewInvoker creates an Invoker; Bind completes it.
func NewInvoker(clients ClientSource) *Invoker { return &Invoker{clients: clients} }
// Bind supplies the tenant configuration and installed plugin rows.
func (iv *Invoker) Bind(t *tenancy.Service, plugins interfaces.PluginRepository) {
iv.mu.Lock()
iv.tenancy, iv.plugins = t, plugins
iv.mu.Unlock()
}
// Default timeouts per call kind, used when the caller set no deadline.
const (
searchTimeout = 10 * time.Second
metadataTimeout = time.Minute
)
// withDefaultTimeout bounds ctx unless it already has a deadline.
func withDefaultTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) {
if _, ok := ctx.Deadline(); ok {
return context.WithCancel(ctx)
}
return context.WithTimeout(ctx, d)
}
// Envelope builds the call envelope for a plugin with an instance config.
func (iv *Invoker) Envelope(
ctx context.Context,
m *manifest.Manifest,
instance map[string]any,
) (pluginapi.Envelope, error) {
env := pluginapi.Envelope{Config: pluginapi.Config{Instance: instance}}
if tenantID, ok := types.TenantIDFromContext(ctx); ok {
env.Context.TenantID = tenantID
}
if userID, ok := ctx.Value(types.UserIDContextKey).(string); ok {
env.Context.UserID = userID
}
if lang, ok := types.LanguageFromContext(ctx); ok {
env.Context.Locale = lang
}
if rid, ok := ctx.Value(types.RequestIDContextKey).(string); ok {
env.Context.RequestID = rid
}
if d, ok := ctx.Deadline(); ok {
env.Context.Deadline = &d
}
iv.mu.RLock()
t, plugins := iv.tenancy, iv.plugins
iv.mu.RUnlock()
if plugins != nil && len(m.Config.SystemSchema) > 0 {
sys, _, err := install.OpenSystemConfig(ctx, plugins, m)
if err != nil {
return env, err
}
env.Config.System = sys
}
if t != nil && env.Context.TenantID != 0 && len(m.Config.TenantSchema) > 0 {
cfg, _, err := t.OpenConfig(ctx, env.Context.TenantID, m.ID)
if err != nil {
return env, err
}
env.Config.Tenant = cfg
}
return env, nil
}
// callError presents a plugin's error by its message, the way WeKnora's own
// errors read in the UI; the protocol code stays reachable through
// errors.As(err, **pluginapi.Error).
type callError struct {
pluginID string
err *pluginapi.Error
}
func (e *callError) Error() string {
if e.err.Code == pluginapi.CodeUnavailable {
return "plugin " + e.pluginID + " is unavailable: " + e.err.Message
}
return e.err.Message
}
func (e *callError) Unwrap() error { return e.err }
func present(pluginID string, err error) error {
if pe, ok := pluginapi.AsError(err); ok {
return &callError{pluginID: pluginID, err: pe}
}
return err
}
// Call makes a unary call to a plugin.
func (iv *Invoker) Call(
ctx context.Context, m *manifest.Manifest, path string, instance map[string]any, input, out any,
) error {
c, err := iv.clients.Client(m.ID)
if err != nil {
return present(m.ID, err)
}
env, err := iv.Envelope(ctx, m, instance)
if err != nil {
return err
}
return present(m.ID, c.Call(ctx, path, env, input, out))
}
// Stream makes a streaming call to a plugin.
func (iv *Invoker) Stream(
ctx context.Context, m *manifest.Manifest, path string, instance map[string]any, input any,
fn func(pluginapi.Event) error,
) ([]byte, error) {
c, err := iv.clients.Client(m.ID)
if err != nil {
return nil, present(m.ID, err)
}
env, err := iv.Envelope(ctx, m, instance)
if err != nil {
return nil, err
}
end, err := c.Stream(ctx, path, env, input, fn)
return end, present(m.ID, err)
}

View File

@ -0,0 +1,278 @@
package activate
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"path"
"sort"
"strings"
"sync"
"gopkg.in/yaml.v3"
infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
"github.com/Tencent/WeKnora/internal/plugin/configschema"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/pkg"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// WebSearch registers code plugins' web search providers as provider types.
// A plugin provider stores its configuration in the same parameters as a
// builtin one: the secret api_key and string fields under extra_config, so
// encryption, redaction and the credentials endpoint work unchanged.
type WebSearch struct {
iv *Invoker
registry *infra_web_search.Registry
mu sync.Mutex
registered map[string][]string // plugin ID → provider type IDs
}
// NewWebSearch creates the web search activator.
func NewWebSearch(iv *Invoker, registry *infra_web_search.Registry) *WebSearch {
return &WebSearch{iv: iv, registry: registry, registered: map[string][]string{}}
}
// Name implements reconcile.Activator.
func (a *WebSearch) Name() string { return "webSearch" }
// Activate implements reconcile.Activator: all of a plugin's providers or
// none.
func (a *WebSearch) Activate(_ context.Context, l *reconcile.Loaded) error {
var ids []string
for _, c := range l.Manifest.Contributes[manifest.PointWebSearch] {
id := manifest.QualifiedID(l.Manifest.ID, c.ID)
pt, err := webSearchType(l, c, id)
if err == nil {
m, local := l.Manifest, c.ID
err = a.registry.RegisterPlugin(
pt,
func(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
return &remoteSearch{
iv: a.iv,
m: m,
local: local,
typeID: id,
instance: webSearchInstance(params),
}, nil
},
)
}
if err != nil {
for _, done := range ids {
a.registry.Unregister(done)
}
return fmt.Errorf("webSearch %s: %w", c.ID, err)
}
ids = append(ids, id)
}
a.mu.Lock()
a.registered[l.Manifest.ID] = ids
a.mu.Unlock()
return nil
}
// Deactivate implements reconcile.Activator.
func (a *WebSearch) Deactivate(_ context.Context, pluginID string) error {
a.mu.Lock()
ids := a.registered[pluginID]
delete(a.registered, pluginID)
a.mu.Unlock()
for _, id := range ids {
a.registry.Unregister(id)
}
return nil
}
// webSearchType describes a plugin provider for the type list.
func webSearchType(l *reconcile.Loaded, c manifest.Contribution, id string) (infra_web_search.PluginType, error) {
src, err := instanceSchema(l.Package, c)
if err != nil {
return infra_web_search.PluginType{}, err
}
schema, requiresKey, err := webSearchSchema(src)
if err != nil {
return infra_web_search.PluginType{}, err
}
icon, err := iconDataURI(l.Package, c.Icon)
if err != nil {
return infra_web_search.PluginType{}, err
}
info := types.WebSearchProviderTypeInfo{
ID: id, Name: c.Name.Default, Names: c.Name.Locales, Description: c.Description.Default,
RequiresAPIKey: requiresKey, Icon: icon, PluginID: l.Manifest.ID, ConfigSchema: schema,
}
validate := func(params types.WebSearchProviderParameters) error {
value := map[string]any{"api_key": params.APIKey}
extra := map[string]any{}
for k, v := range params.ExtraConfig {
extra[k] = v
}
value["extra_config"] = extra
if errs := configschema.Validate(schema, value); len(errs) > 0 {
return errs
}
return nil
}
return infra_web_search.PluginType{Info: info, Validate: validate}, nil
}
// webSearchSchema maps a plugin's instance schema onto the provider
// parameters: api_key stays top-level (the only secret), everything else
// moves under extra_config, which stores strings.
func webSearchSchema(src *configschema.Schema) (*configschema.Schema, bool, error) {
out := configschema.Object()
if src == nil {
return out, false, nil
}
required := map[string]bool{}
for _, k := range src.Required {
required[k] = true
}
extra := &configschema.Schema{
Type: configschema.TypeObject, Group: "credentials", Order: 40, Properties: map[string]*configschema.Schema{},
}
var keys []string
for k := range src.Properties {
keys = append(keys, k)
}
sort.Strings(keys)
requiresKey := false
for _, k := range keys {
prop := *src.Properties[k]
if k == "api_key" {
prop.Type, prop.Secret, prop.Group = configschema.TypeString, true, "credentials"
out.Set("api_key", &prop, required[k])
requiresKey = required[k]
continue
}
if prop.Secret {
return nil, false, fmt.Errorf("instanceSchema field %s is secret; only api_key may be", k)
}
if prop.Type != "" && prop.Type != configschema.TypeString {
return nil, false, fmt.Errorf(
"instanceSchema field %s must be a string (web search settings are stored as strings)",
k,
)
}
prop.Type, prop.Group = configschema.TypeString, ""
extra.Set(k, &prop, required[k])
}
if len(extra.Properties) > 0 {
out.Set("extra_config", extra, len(extra.Required) > 0)
}
return out, requiresKey, nil
}
// webSearchInstance is what the plugin receives as config.instance.
func webSearchInstance(params types.WebSearchProviderParameters) map[string]any {
out := map[string]any{}
if params.APIKey != "" {
out["api_key"] = params.APIKey
}
for k, v := range params.ExtraConfig {
out[k] = v
}
return out
}
// instanceSchema reads a contribution's instanceSchema file (JSON or YAML).
func instanceSchema(p *pkg.Package, c manifest.Contribution) (*configschema.Schema, error) {
if c.InstanceSchema == "" || p == nil {
return nil, nil
}
raw, ok := p.ReadFile(c.InstanceSchema)
if !ok {
return nil, fmt.Errorf("instanceSchema %s is not in the package", c.InstanceSchema)
}
var v any
if err := yaml.Unmarshal(raw, &v); err != nil {
return nil, fmt.Errorf("instanceSchema %s: %w", c.InstanceSchema, err)
}
b, err := json.Marshal(v)
if err != nil {
return nil, err
}
s, err := configschema.Parse(b)
if err != nil {
return nil, fmt.Errorf("instanceSchema %s: %w", c.InstanceSchema, err)
}
return s, nil
}
// maxIconBytes caps an icon inlined into type listings.
const maxIconBytes = 64 << 10
// iconDataURI inlines a package icon (SVG or PNG) as a data: URI.
func iconDataURI(p *pkg.Package, file string) (string, error) {
if file == "" || p == nil {
return "", nil
}
b, ok := p.ReadFile(file)
if !ok {
return "", fmt.Errorf("icon %s is not in the package", file)
}
if len(b) > maxIconBytes {
return "", fmt.Errorf("icon %s is over %d bytes", file, maxIconBytes)
}
var mime string
switch strings.ToLower(path.Ext(file)) {
case ".svg":
mime = "image/svg+xml"
case ".png":
mime = "image/png"
default:
return "", fmt.Errorf("icon %s must be .svg or .png", file)
}
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b), nil
}
// remoteSearch is one configured plugin provider.
type remoteSearch struct {
iv *Invoker
m *manifest.Manifest
local string
typeID string
instance map[string]any
}
// Name implements interfaces.WebSearchProvider.
func (r *remoteSearch) Name() string { return r.typeID }
// Search implements interfaces.WebSearchProvider.
func (r *remoteSearch) Search(
ctx context.Context, query string, maxResults int, includeDate bool,
) ([]*types.WebSearchResult, error) {
return r.SearchWithFilters(ctx, query, maxResults, includeDate, types.WebSearchFilters{})
}
// SearchWithFilters implements interfaces.FilteredWebSearchProvider; the
// plugin must fail rather than ignore a filter it cannot honour.
func (r *remoteSearch) SearchWithFilters(
ctx context.Context, query string, maxResults int, includeDate bool, filters types.WebSearchFilters,
) ([]*types.WebSearchResult, error) {
ctx, cancel := withDefaultTimeout(ctx, searchTimeout)
defer cancel()
in := pluginapi.SearchInput{
Query: query, MaxResults: maxResults, IncludeDate: includeDate,
Region: filters.Country, Freshness: filters.Freshness,
}
var out pluginapi.SearchOutput
if err := r.iv.Call(ctx, r.m, pluginapi.SearchPath(r.local), r.instance, in, &out); err != nil {
return nil, err
}
results := make([]*types.WebSearchResult, 0, len(out.Results))
for _, h := range out.Results {
results = append(results, &types.WebSearchResult{
Title: h.Title, URL: h.URL, Snippet: h.Snippet, Content: h.Content,
Source: r.typeID, Age: h.Age, PublishedAt: h.PublishedAt,
})
}
return results, nil
}

View File

@ -0,0 +1,152 @@
package activate
import (
"context"
"fmt"
"net/http/httptest"
"strings"
"testing"
infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
"github.com/Tencent/WeKnora/internal/plugin/pkg"
"github.com/Tencent/WeKnora/internal/plugin/plugintest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
const searchPluginManifest = `schemaVersion: 1
id: acme.search
version: 1.0.0
apiVersion: weknora.plugin/v1
name: { en-US: ACME Search }
publisher: { id: acme }
runtime: { type: host, kind: binary, entry: bin/search }
contributes:
webSearch:
- id: brave
name: { en-US: ACME Brave, zh-CN: ACME 勇敢搜索 }
icon: icons/brave.svg
instanceSchema: schemas/brave.yaml
`
// fakeClients serves one plugin from an in-process SDK handler.
type fakeClients struct{ c *client.Client }
func (f fakeClients) Client(string) (*client.Client, error) { return f.c, nil }
func searchLoaded(t *testing.T, schema string) *reconcile.Loaded {
t.Helper()
p, err := pkg.Open(plugintest.Zip(t, map[string]string{
"plugin.yaml": searchPluginManifest,
"bin/search": "x",
"icons/brave.svg": `<svg xmlns="http://www.w3.org/2000/svg"/>`,
"schemas/brave.yaml": schema,
}))
if err != nil {
t.Fatal(err)
}
return &reconcile.Loaded{Manifest: p.Manifest, Package: p}
}
const braveSchema = `type: object
required: [api_key, market]
properties:
api_key: { type: string, title: API key }
market: { type: string, enum: [en-US, zh-CN], default: en-US }
`
func TestPluginWebSearchProvider(t *testing.T) {
ctx := context.Background()
plugin := pluginsdk.New(pluginsdk.Info{ID: "acme.search", Version: "1.0.0"})
plugin.WebSearch(
"brave",
pluginsdk.WebSearchFunc(
func(_ context.Context, call *pluginsdk.Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
if call.Config.Instance["api_key"] != "k-1" || call.Config.Instance["market"] != "zh-CN" {
return nil, pluginapi.Errorf(
pluginapi.CodeUnauthorized,
"bad instance config %v",
call.Config.Instance,
)
}
if in.Region != "" && in.Region != "CN" {
return nil, pluginapi.Errorf(pluginapi.CodeInvalidConfig, "region %s unsupported", in.Region)
}
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{
Title: in.Query, URL: fmt.Sprint("https://acme.example/?tenant=", call.TenantID),
}}}, nil
},
),
)
srv := httptest.NewServer(plugin.Handler())
defer srv.Close()
registry := infra_web_search.NewRegistry()
a := NewWebSearch(NewInvoker(fakeClients{client.New(srv.URL, nil, nil)}), registry)
if err := a.Activate(ctx, searchLoaded(t, braveSchema)); err != nil {
t.Fatalf("Activate: %v", err)
}
pt, ok := registry.PluginType("acme.search/brave")
if !ok {
t.Fatal("type not registered")
}
info := pt.Info
if info.Names["zh-CN"] != "ACME 勇敢搜索" || !info.RequiresAPIKey ||
!strings.HasPrefix(info.Icon, "data:image/svg+xml;base64,") ||
info.PluginID != "acme.search" {
t.Fatalf("info = %+v", info)
}
if ak := info.ConfigSchema.Properties["api_key"]; ak == nil || !ak.Secret || ak.Group != "credentials" {
t.Fatalf("api_key schema = %+v", ak)
}
if ec := info.ConfigSchema.Properties["extra_config"]; ec == nil || ec.Properties["market"] == nil {
t.Fatalf("extra_config schema = %+v", ec)
}
if err := pt.Validate(types.WebSearchProviderParameters{APIKey: "k"}); err == nil {
t.Fatal("a missing required extra_config field must fail validation")
}
params := types.WebSearchProviderParameters{APIKey: "k-1", ExtraConfig: map[string]string{"market": "zh-CN"}}
if err := pt.Validate(params); err != nil {
t.Fatalf("valid params rejected: %v", err)
}
provider, err := registry.CreateProvider("acme.search/brave", params)
if err != nil {
t.Fatal(err)
}
tctx := context.WithValue(ctx, types.TenantIDContextKey, uint64(7))
res, err := provider.Search(tctx, "weknora", 3, false)
if err != nil || len(res) != 1 || res[0].Title != "weknora" || res[0].URL != "https://acme.example/?tenant=7" ||
res[0].Source != "acme.search/brave" {
t.Fatalf("search = %+v, %v", res, err)
}
filtered := provider.(interfaces.FilteredWebSearchProvider)
if _, err := filtered.SearchWithFilters(tctx, "q", 3, false, types.WebSearchFilters{Country: "DE"}); err == nil ||
!strings.Contains(err.Error(), "region DE unsupported") {
t.Fatalf("an unsupported filter must fail, got %v", err)
}
if err := a.Deactivate(ctx, "acme.search"); err != nil {
t.Fatal(err)
}
if _, ok := registry.PluginType("acme.search/brave"); ok {
t.Fatal("type survived Deactivate")
}
}
func TestPluginWebSearchSchemaRules(t *testing.T) {
a := NewWebSearch(NewInvoker(fakeClients{}), infra_web_search.NewRegistry())
for name, schema := range map[string]string{
"secret field": "type: object\nproperties: { token: { type: string, x-secret: true } }\n",
"number field": "type: object\nproperties: { limit: { type: integer } }\n",
} {
if err := a.Activate(context.Background(), searchLoaded(t, schema)); err == nil {
t.Errorf("%s: want an activation error", name)
}
}
}

View File

@ -0,0 +1,140 @@
// Package host is the embedded plugin host: it runs host-runtime plugins as
// child processes of this WeKnora node and hands out clients to reach them.
// Each node runs its own copy of every enabled host plugin (the reconciler
// activates it everywhere), so a call never leaves the node.
//
// A plugin process gets a private socket, a random token, an environment
// built from scratch (no WeKnora secrets) and an egress proxy limited to the
// hosts its manifest was granted. The host checks that the process is the
// installed package, health-checks it and restarts it with backoff.
package host
import (
"context"
"fmt"
"sync"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// StateReporter learns when a running plugin's health changes, so the
// cluster view shows a crashed plugin as degraded.
type StateReporter interface {
ReportRuntime(pluginID string, healthy bool, err error)
}
// Manager runs host plugins. It is a reconcile.Activator and must come
// before the activators that route calls to the plugins.
type Manager struct {
reporter StateReporter
mu sync.Mutex
procs map[string]*process
}
// NewManager creates an empty host.
func NewManager() *Manager { return &Manager{procs: map[string]*process{}} }
// SetReporter wires health changes to the reconciler.
func (m *Manager) SetReporter(r StateReporter) {
m.mu.Lock()
m.reporter = r
m.mu.Unlock()
}
// Name implements reconcile.Activator.
func (m *Manager) Name() string { return "host" }
// ActivatesInPlace implements reconcile.InPlaceActivator: an upgrade starts
// the new process before the old one stops.
func (m *Manager) ActivatesInPlace() {}
// Activate starts a host plugin and returns once it is ready. Other runtimes
// are ignored. A new version replaces the old process only after it started.
func (m *Manager) Activate(ctx context.Context, l *reconcile.Loaded) error {
if l.Manifest.Runtime.Type != manifest.RuntimeHost {
return nil
}
if l.Manifest.Runtime.Kind != "binary" {
return fmt.Errorf("runtime.kind %q is not supported by this host yet; only binary", l.Manifest.Runtime.Kind)
}
id := l.Manifest.ID
p, err := startProcess(spec{m: l.Manifest, dir: l.Dir}, func(s State, err error) {
m.report(id, s, err)
})
if err != nil {
return err
}
m.mu.Lock()
old := m.procs[id]
m.procs[id] = p
m.mu.Unlock()
if old != nil {
old.stop()
}
logger.Infof(ctx, "[plugin] host started %s %s", id, l.Manifest.Version)
return nil
}
// Deactivate implements reconcile.Activator: it stops the process.
func (m *Manager) Deactivate(ctx context.Context, pluginID string) error {
m.mu.Lock()
p := m.procs[pluginID]
delete(m.procs, pluginID)
m.mu.Unlock()
if p != nil {
p.stop()
logger.Infof(ctx, "[plugin] host stopped %s", pluginID)
}
return nil
}
func (m *Manager) report(pluginID string, s State, err error) {
m.mu.Lock()
r := m.reporter
m.mu.Unlock()
if r == nil {
return
}
switch s {
case StateReady:
r.ReportRuntime(pluginID, true, nil)
case StateDegraded:
r.ReportRuntime(pluginID, false, err)
}
}
// Client returns a client for a running plugin. The error is a retryable
// pluginapi unavailable error while the plugin is starting or restarting.
func (m *Manager) Client(pluginID string) (*client.Client, error) {
m.mu.Lock()
p := m.procs[pluginID]
m.mu.Unlock()
if p == nil {
return nil, &pluginapi.Error{
Code: pluginapi.CodeUnavailable, Message: fmt.Sprintf("plugin %s is not running on this node", pluginID),
}
}
return p.Client()
}
// Close stops every plugin, for shutdown.
func (m *Manager) Close() {
m.mu.Lock()
procs := m.procs
m.procs = map[string]*process{}
m.mu.Unlock()
var wg sync.WaitGroup
for _, p := range procs {
wg.Add(1)
go func() {
defer wg.Done()
p.stop()
}()
}
wg.Wait()
}

View File

@ -0,0 +1,297 @@
package host
import (
"context"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
var (
buildOnce sync.Once
binary string
buildErr error
)
// echoBinary builds testdata/echoplugin once per test run.
func echoBinary(t *testing.T) string {
t.Helper()
buildOnce.Do(func() {
dir, err := os.MkdirTemp("", "echoplugin")
if err != nil {
buildErr = err
return
}
binary = filepath.Join(dir, "echo")
out, err := exec.Command("go", "build", "-o", binary, "./testdata/echoplugin").CombinedOutput()
if err != nil {
buildErr = err
t.Logf("%s", out)
}
})
if buildErr != nil {
t.Fatalf("build echo plugin: %v", buildErr)
}
return binary
}
// install lays out an extracted package with the echo binary.
func install(t *testing.T, version string, fakeVersion string) *reconcile.Loaded {
t.Helper()
dir := t.TempDir()
bin := filepath.Join(dir, "bin", runtime.GOOS+"-"+runtime.GOARCH)
if err := os.MkdirAll(bin, 0o755); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(echoBinary(t))
if err != nil {
t.Fatal(err)
}
// Extraction writes plain files; the host must make the entry runnable.
name := "echo"
if runtime.GOOS == "windows" {
name += ".exe"
}
if err := os.WriteFile(filepath.Join(bin, name), data, 0o644); err != nil {
t.Fatal(err)
}
if fakeVersion != "" {
_ = os.WriteFile(filepath.Join(dir, "version"), []byte(fakeVersion), 0o644)
}
m := &manifest.Manifest{
SchemaVersion: manifest.SchemaVersion, ID: "acme.echo", Version: version, APIVersion: pluginapi.APIVersion,
Name: manifest.Text("Echo", nil), Publisher: manifest.Publisher{ID: "acme"},
Runtime: manifest.Runtime{Type: manifest.RuntimeHost, Kind: "binary", Entry: "bin/{os}-{arch}/echo"},
Contributes: manifest.Contributions{
manifest.PointWebSearch: {{ID: "echo", Name: manifest.Text("Echo", nil)}},
},
}
return &reconcile.Loaded{Manifest: m, Dir: dir}
}
type reports struct {
mu sync.Mutex
log []string
}
func (r *reports) ReportRuntime(id string, healthy bool, _ error) {
r.mu.Lock()
defer r.mu.Unlock()
r.log = append(r.log, id+":"+strconv.FormatBool(healthy))
}
func (r *reports) String() string {
r.mu.Lock()
defer r.mu.Unlock()
return strings.Join(r.log, ",")
}
func fastTimings(t *testing.T) {
t.Helper()
oldBackoff, oldInterval, oldGrace := restartBackoffFloor, healthInterval, stopGrace
restartBackoffFloor, healthInterval, stopGrace = 20*time.Millisecond, 100*time.Millisecond, 2*time.Second
t.Cleanup(func() { restartBackoffFloor, healthInterval, stopGrace = oldBackoff, oldInterval, oldGrace })
}
func search(t *testing.T, m *Manager, q string) (string, error) {
t.Helper()
c, err := m.Client("acme.echo")
if err != nil {
return "", err
}
var out pluginapi.SearchOutput
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err = c.Call(ctx, pluginapi.SearchPath("echo"), pluginapi.Envelope{}, pluginapi.SearchInput{Query: q}, &out)
if err != nil {
return "", err
}
return out.Results[0].Title, nil
}
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(20 * time.Millisecond)
}
}
func TestHostRunsRestartsAndStopsAPlugin(t *testing.T) {
fastTimings(t)
t.Setenv("DB_PASSWORD", "must-not-leak")
ctx := context.Background()
m := NewManager()
rep := &reports{}
m.SetReporter(rep)
defer m.Close()
if err := m.Activate(ctx, install(t, "1.0.0", "")); err != nil {
t.Fatalf("Activate: %v", err)
}
if got, err := search(t, m, "hello"); err != nil || got != "hello" {
t.Fatalf("search = %q, %v", got, err)
}
env, err := search(t, m, "env")
if err != nil {
t.Fatal(err)
}
if strings.Contains(env, "DB_PASSWORD") || !strings.Contains(env, "HTTPS_PROXY") ||
!strings.Contains(env, pluginapi.EnvToken) {
t.Fatalf("plugin environment = %s", env)
}
pid1, _ := search(t, m, "pid")
_, _ = search(t, m, "crash")
waitFor(t, "restart", func() bool {
pid, err := search(t, m, "pid")
return err == nil && pid != pid1
})
if !strings.Contains(rep.String(), "acme.echo:false") || !strings.HasSuffix(rep.String(), "acme.echo:true") {
t.Fatalf("reports = %s", rep)
}
pid2, _ := search(t, m, "pid")
if err := m.Deactivate(ctx, "acme.echo"); err != nil {
t.Fatal(err)
}
if _, err := m.Client("acme.echo"); err == nil {
t.Fatal("a stopped plugin must have no client")
}
if runtime.GOOS != "windows" {
n, _ := strconv.Atoi(pid2)
if proc, err := os.FindProcess(n); err == nil && proc.Signal(syscall.Signal(0)) == nil {
t.Fatalf("process %d still runs after Deactivate", n)
}
}
}
func TestHostRefusesAProcessThatIsNotThePackage(t *testing.T) {
fastTimings(t)
m := NewManager()
defer m.Close()
err := m.Activate(context.Background(), install(t, "1.0.0", "9.9.9"))
if err == nil || !strings.Contains(err.Error(), "reports acme.echo@9.9.9") {
t.Fatalf("want a manifest mismatch, got %v", err)
}
l := install(t, "1.0.0", "")
l.Manifest.Contributes[manifest.PointWebSearch] = append(l.Manifest.Contributes[manifest.PointWebSearch],
manifest.Contribution{ID: "missing", Name: manifest.Text("Missing", nil)})
if err := m.Activate(context.Background(), l); err == nil || !strings.Contains(err.Error(), "webSearch/missing") {
t.Fatalf("want a missing contribution error, got %v", err)
}
l = install(t, "1.0.0", "")
l.Manifest.Runtime.Entry = "bin/{os}-{arch}/nope"
if err := m.Activate(context.Background(), l); err == nil || !strings.Contains(err.Error(), "has no") {
t.Fatalf("want a missing build error, got %v", err)
}
}
func TestUpgradeSwapsProcessesInPlace(t *testing.T) {
fastTimings(t)
ctx := context.Background()
m := NewManager()
defer m.Close()
if err := m.Activate(ctx, install(t, "1.0.0", "")); err != nil {
t.Fatal(err)
}
pid1, _ := search(t, m, "pid")
if err := m.Activate(ctx, install(t, "1.1.0", "")); err != nil {
t.Fatal(err)
}
pid2, err := search(t, m, "pid")
if err != nil || pid2 == pid1 {
t.Fatalf("after upgrade pid = %s (was %s), %v", pid2, pid1, err)
}
}
func TestEgressProxyEnforcesThePolicy(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("hello from " + r.Host))
}))
defer upstream.Close()
p, err := startEgressProxy("acme.echo", []string{"api.allowed.test", "*.wild.test"}, t.Logf)
if err != nil {
t.Fatal(err)
}
defer p.Close()
// Every name resolves to the test server; the policy is what differs.
p.dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, upstream.Listener.Addr().String())
}
proxyURL, _ := url.Parse(p.URL())
c := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
for host, want := range map[string]int{
"api.allowed.test": 200, "x.wild.test": 200, "wild.test": 403, "evil.test": 403,
} {
resp, err := c.Get("http://" + host + "/")
if err != nil {
t.Fatalf("%s: %v", host, err)
}
_ = resp.Body.Close()
if resp.StatusCode != want {
t.Errorf("%s: status %d, want %d", host, resp.StatusCode, want)
}
}
}
func TestEgressProxyTunnelsHTTPS(t *testing.T) {
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("secure"))
}))
defer upstream.Close()
p, err := startEgressProxy("acme.echo", []string{"*.wild.test"}, t.Logf)
if err != nil {
t.Fatal(err)
}
defer p.Close()
p.dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, network, upstream.Listener.Addr().String())
}
proxyURL, _ := url.Parse(p.URL())
tr := upstream.Client().Transport.(*http.Transport).Clone()
tr.Proxy = http.ProxyURL(proxyURL)
tr.TLSClientConfig.InsecureSkipVerify = true
c := &http.Client{Transport: tr}
resp, err := c.Get("https://api.wild.test/")
if err != nil {
t.Fatal(err)
}
body := make([]byte, 16)
n, _ := resp.Body.Read(body)
_ = resp.Body.Close()
if string(body[:n]) != "secure" {
t.Fatalf("tunnelled body = %q", body[:n])
}
if _, err := c.Get("https://evil.test/"); err == nil || !strings.Contains(err.Error(), "Forbidden") {
t.Fatalf("CONNECT to a host outside the policy must be refused, got %v", err)
}
}
func TestProtocolVersionsAgree(t *testing.T) {
if manifest.ExtensionAPIVersion != pluginapi.APIVersion {
t.Fatalf("manifest accepts %q, the protocol package speaks %q",
manifest.ExtensionAPIVersion, pluginapi.APIVersion)
}
}

View File

@ -0,0 +1,16 @@
//go:build linux
package host
import (
"os/exec"
"syscall"
)
// configureChild puts the plugin in its own process group and has the kernel
// kill it if WeKnora dies, so a crash never leaves orphaned plugins.
func configureChild(cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
}
func terminate(cmd *exec.Cmd) error { return cmd.Process.Signal(syscall.SIGTERM) }

View File

@ -0,0 +1,16 @@
//go:build unix && !linux
package host
import (
"os/exec"
"syscall"
)
// configureChild puts the plugin in its own process group, so a signal
// meant for WeKnora's terminal does not reach it directly.
func configureChild(cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
}
func terminate(cmd *exec.Cmd) error { return cmd.Process.Signal(syscall.SIGTERM) }

View File

@ -0,0 +1,11 @@
//go:build windows
package host
import "os/exec"
func configureChild(*exec.Cmd) {}
// terminate kills the plugin: Windows has no SIGTERM for console-less
// children.
func terminate(cmd *exec.Cmd) error { return cmd.Process.Kill() }

View File

@ -0,0 +1,438 @@
package host
import (
"bufio"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Timing of a host plugin's life. Variables so tests can shrink them.
var (
handshakeTimeout = 30 * time.Second
healthInterval = 15 * time.Second
healthTimeout = 5 * time.Second
unhealthyLimit = 3
stopGrace = 10 * time.Second
restartBackoffFloor = time.Second
restartBackoffCeiling = time.Minute
)
// State of a supervised process.
type State string
// Process states reported to the reconciler.
const (
StateStarting State = "starting"
StateReady State = "ready"
StateDegraded State = "degraded"
StateStopped State = "stopped"
)
// spec is everything needed to start one plugin version.
type spec struct {
m *manifest.Manifest
dir string // extracted package
}
// entryPath resolves runtime.entry for this machine inside the package.
func entryPath(m *manifest.Manifest, dir string) (string, error) {
rel := strings.NewReplacer("{os}", runtime.GOOS, "{arch}", runtime.GOARCH).Replace(m.Runtime.Entry)
if runtime.GOOS == "windows" && filepath.Ext(rel) == "" {
rel += ".exe"
}
p, err := utils.SafeJoinUnderBase(dir, rel)
if err != nil {
return "", fmt.Errorf("runtime.entry %q: %w", m.Runtime.Entry, err)
}
info, err := os.Stat(p)
if err != nil {
return "", fmt.Errorf("the package has no %s build (%s)", runtime.GOOS+"/"+runtime.GOARCH, rel)
}
if !info.Mode().IsRegular() {
return "", fmt.Errorf("runtime.entry %s is not a file", rel)
}
// Packages are extracted without modes; the entry has to be executable.
if err := os.Chmod(p, 0o755); err != nil {
return "", err
}
return p, nil
}
// process is one running plugin version and its supervisor.
type process struct {
spec spec
onState func(State, error)
mu sync.RWMutex
client *client.Client
state State
lastErr error
cancel context.CancelFunc
done chan struct{}
proxy *egressProxy
sockDir string
}
// launched is one started child.
type launched struct {
cmd *exec.Cmd
client *client.Client
exited chan error
}
// startProcess starts the plugin, waits until it is ready and keeps it
// running until stop. It returns once the first start succeeded or failed.
func startProcess(sp spec, onState func(State, error)) (*process, error) {
entry, err := entryPath(sp.m, sp.dir)
if err != nil {
return nil, err
}
sockDir, err := socketDir()
if err != nil {
return nil, err
}
proxy, err := startEgressProxy(sp.m.ID, sp.m.Permissions.Egress, func(f string, a ...any) {
logger.Warnf(context.Background(), f, a...)
})
if err != nil {
_ = os.RemoveAll(sockDir)
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
p := &process{spec: sp, onState: onState, cancel: cancel, done: make(chan struct{}), proxy: proxy, sockDir: sockDir}
p.setState(StateStarting, nil)
first, err := p.launch(ctx, entry)
if err != nil {
cancel()
proxy.Close()
_ = os.RemoveAll(sockDir)
close(p.done)
p.setState(StateStopped, err)
return nil, err
}
// Ready before Activate returns: callers route to the plugin right away.
p.setClient(first.client)
p.setState(StateReady, nil)
go p.supervise(ctx, entry, first)
return p, nil
}
// socketDir makes a short private directory for the socket: unix socket
// paths are limited to about 100 bytes.
func socketDir() (string, error) {
base := os.Getenv("WEKNORA_PLUGIN_SOCKET_DIR")
if base == "" {
base = os.TempDir()
}
return os.MkdirTemp(base, "wkp")
}
func randomToken() string {
b := make([]byte, 32)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
// childEnv is the plugin's whole environment. It is built from scratch so
// WeKnora's own secrets (database, AES key, vendor keys) never reach plugin
// code.
func (p *process) childEnv(socket, token string) []string {
env := []string{
pluginapi.EnvSocket + "=" + socket,
pluginapi.EnvNetwork + "=unix",
pluginapi.EnvToken + "=" + token,
pluginapi.EnvPluginID + "=" + p.spec.m.ID,
pluginapi.EnvPluginVersion + "=" + p.spec.m.Version,
"HOME=" + p.sockDir,
"TMPDIR=" + p.sockDir,
"HTTP_PROXY=" + p.proxy.URL(),
"HTTPS_PROXY=" + p.proxy.URL(),
"http_proxy=" + p.proxy.URL(),
"https_proxy=" + p.proxy.URL(),
"NO_PROXY=",
"no_proxy=",
}
for _, k := range []string{"PATH", "LANG", "LC_ALL", "TZ", "SYSTEMROOT", "WINDIR"} {
if v, ok := os.LookupEnv(k); ok {
env = append(env, k+"="+v)
}
}
return env
}
// launch starts the child and waits for handshake, health and a manifest
// that matches the installed package.
func (p *process) launch(ctx context.Context, entry string) (*launched, error) {
socket := filepath.Join(p.sockDir, "p.sock")
_ = os.Remove(socket)
token := randomToken()
cmd := exec.Command(entry)
cmd.Dir = p.spec.dir
cmd.Env = p.childEnv(socket, token)
configureChild(cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("start %s: %w", filepath.Base(entry), err)
}
id := p.spec.m.ID + "@" + p.spec.m.Version
handshake := make(chan pluginapi.Handshake, 1)
hsErr := make(chan error, 1)
go forwardLogs(stderr, id, nil, nil)
go forwardLogs(stdout, id, handshake, hsErr)
exited := make(chan error, 1)
go func() { exited <- cmd.Wait() }()
kill := func() {
stopChild(cmd, exited)
}
timer := time.NewTimer(handshakeTimeout)
defer timer.Stop()
var hs pluginapi.Handshake
select {
case hs = <-handshake:
case err := <-hsErr:
kill()
return nil, err
case err := <-exited:
exited <- err // keep it for supervise
return nil, fmt.Errorf("plugin exited before it was ready: %v", err)
case <-timer.C:
kill()
return nil, fmt.Errorf("plugin did not print its handshake within %s", handshakeTimeout)
case <-ctx.Done():
kill()
return nil, ctx.Err()
}
if hs.Network == "unix" && hs.Address != socket {
kill()
return nil, fmt.Errorf("plugin listens on %s, not the socket it was given", hs.Address)
}
c := client.ForHandshake(hs, client.Bearer(token))
cctx, cancel := context.WithTimeout(ctx, healthTimeout)
defer cancel()
if err := c.Health(cctx); err != nil {
c.Close()
kill()
return nil, fmt.Errorf("plugin is not healthy: %w", err)
}
m, err := c.Manifest(cctx)
if err != nil {
c.Close()
kill()
return nil, fmt.Errorf("read plugin manifest: %w", err)
}
if err := checkServedManifest(p.spec.m, m); err != nil {
c.Close()
kill()
return nil, err
}
return &launched{cmd: cmd, client: c, exited: exited}, nil
}
// checkServedManifest makes sure the process is the package that was
// installed and serves what the manifest promised.
func checkServedManifest(want *manifest.Manifest, got *pluginapi.Manifest) error {
if got.ID != want.ID || got.Version != want.Version {
return fmt.Errorf("process reports %s@%s, the package is %s@%s", got.ID, got.Version, want.ID, want.Version)
}
if got.APIVersion != pluginapi.APIVersion {
return fmt.Errorf("process speaks %q, this WeKnora speaks %q", got.APIVersion, pluginapi.APIVersion)
}
for point, contribs := range want.Contributes {
if info, ok := manifest.LookupPoint(point); !ok || info.Declarative {
continue
}
served := map[string]bool{}
for _, id := range got.Contributes[string(point)] {
served[id] = true
}
for _, c := range contribs {
if !served[c.ID] {
return fmt.Errorf("plugin.yaml declares %s/%s but the process does not serve it", point, c.ID)
}
}
}
return nil
}
// forwardLogs copies a child's output into WeKnora's logs. On stdout it
// also watches for the handshake line.
func forwardLogs(r io.Reader, id string, handshake chan<- pluginapi.Handshake, hsErr chan<- error) {
sc := bufio.NewScanner(r)
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
seen := false
for sc.Scan() {
line := sc.Text()
if handshake != nil && !seen {
hs, ok, err := pluginapi.ParseHandshake(line)
if ok {
seen = true
if err != nil {
hsErr <- err
} else {
handshake <- hs
}
continue
}
}
logger.Infof(context.Background(), "[plugin %s] %s", id, line)
}
}
func (p *process) setState(s State, err error) {
p.mu.Lock()
changed := p.state != s || (err != nil) != (p.lastErr != nil)
p.state, p.lastErr = s, err
p.mu.Unlock()
if changed && p.onState != nil {
p.onState(s, err)
}
}
func (p *process) setClient(c *client.Client) {
p.mu.Lock()
old := p.client
p.client = c
p.mu.Unlock()
if old != nil && old != c {
old.Close()
}
}
// Client returns the client of the running child, or an unavailable error.
func (p *process) Client() (*client.Client, error) {
p.mu.RLock()
defer p.mu.RUnlock()
if p.client == nil || p.state != StateReady {
msg := fmt.Sprintf("plugin %s is %s", p.spec.m.ID, p.state)
if p.lastErr != nil {
msg += ": " + p.lastErr.Error()
}
return nil, &pluginapi.Error{Code: pluginapi.CodeUnavailable, Message: msg, Retryable: true}
}
return p.client, nil
}
// supervise keeps the plugin running: health checks, restart with backoff
// after a crash or repeated failed checks.
func (p *process) supervise(ctx context.Context, entry string, cur *launched) {
defer close(p.done)
backoff := restartBackoffFloor
for {
p.setClient(cur.client)
p.setState(StateReady, nil)
exitErr := p.watch(ctx, cur)
p.setClient(nil)
if ctx.Err() != nil {
p.setState(StateStopped, nil)
return
}
logger.Warnf(ctx, "[plugin] %s stopped unexpectedly: %v; restarting in %s", p.spec.m.ID, exitErr, backoff)
p.setState(StateDegraded, exitErr)
for {
select {
case <-ctx.Done():
p.setState(StateStopped, nil)
return
case <-time.After(backoff):
}
backoff = min(backoff*2, restartBackoffCeiling)
next, err := p.launch(ctx, entry)
if err == nil {
cur = next
backoff = restartBackoffFloor
logger.Infof(ctx, "[plugin] %s restarted", p.spec.m.ID)
break
}
if ctx.Err() != nil {
p.setState(StateStopped, nil)
return
}
logger.Warnf(ctx, "[plugin] %s restart failed: %v; next try in %s", p.spec.m.ID, err, backoff)
p.setState(StateDegraded, err)
}
}
}
// watch returns when the child exits, fails its health checks, or ctx ends
// (then it stops the child).
func (p *process) watch(ctx context.Context, cur *launched) error {
t := time.NewTicker(healthInterval)
defer t.Stop()
failures := 0
for {
select {
case err := <-cur.exited:
if err == nil {
err = errors.New("exited")
}
return err
case <-ctx.Done():
stopChild(cur.cmd, cur.exited)
return nil
case <-t.C:
hctx, cancel := context.WithTimeout(ctx, healthTimeout)
err := cur.client.Health(hctx)
cancel()
if err == nil {
failures = 0
continue
}
failures++
if failures >= unhealthyLimit {
stopChild(cur.cmd, cur.exited)
return fmt.Errorf("failed %d health checks: %w", failures, err)
}
}
}
}
// stop ends the plugin and waits for it to exit.
func (p *process) stop() {
p.cancel()
<-p.done
p.proxy.Close()
_ = os.RemoveAll(p.sockDir)
}
// stopChild asks the child to exit, then kills it after stopGrace. exited
// is the channel its Wait result arrives on.
func stopChild(cmd *exec.Cmd, exited chan error) {
if cmd.Process == nil {
return
}
_ = terminate(cmd)
select {
case err := <-exited:
exited <- err
case <-time.After(stopGrace):
_ = cmd.Process.Kill()
err := <-exited
exited <- err
}
}

View File

@ -0,0 +1,192 @@
package host
import (
"context"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/utils"
)
// egressPolicy decides which hosts a plugin may reach: the manifest's
// permissions.egress patterns ("api.example.com", "*.atlassian.net"), which
// the system administrator accepted at install time.
type egressPolicy struct {
any bool // "*": every public host
exact map[string]bool
suffixes []string // ".atlassian.net"
}
func newEgressPolicy(patterns []string) egressPolicy {
p := egressPolicy{exact: map[string]bool{}}
for _, raw := range patterns {
pat := strings.ToLower(strings.TrimSpace(raw))
if pat == "" {
continue
}
if pat == "*" {
p.any = true
continue
}
if rest, ok := strings.CutPrefix(pat, "*."); ok {
p.suffixes = append(p.suffixes, "."+rest)
continue
}
p.exact[pat] = true
}
return p
}
func (p egressPolicy) allows(host string) bool {
if p.any {
// The SSRF-safe dialer still refuses private addresses.
return true
}
host = strings.ToLower(strings.TrimSuffix(host, "."))
if p.exact[host] {
return true
}
for _, s := range p.suffixes {
if strings.HasSuffix(host, s) {
return true
}
}
return false
}
// egressProxy is the HTTP proxy a host plugin's traffic goes through
// (HTTP_PROXY / HTTPS_PROXY in its environment). It forwards only to hosts
// the plugin was granted and dials through the SSRF-safe dialer, so a
// granted name that resolves to a private address is still refused unless
// the operator whitelisted it. A plugin can ignore the proxy variables; on
// Linux a network namespace can make the proxy the only way out.
type egressProxy struct {
pluginID string
policy egressPolicy
ln net.Listener
srv *http.Server
dial func(ctx context.Context, network, addr string) (net.Conn, error)
logf func(format string, args ...any)
wg sync.WaitGroup
}
func startEgressProxy(pluginID string, patterns []string, logf func(string, ...any)) (*egressProxy, error) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, fmt.Errorf("start egress proxy: %w", err)
}
p := &egressProxy{
pluginID: pluginID, policy: newEgressPolicy(patterns), ln: ln,
dial: utils.SSRFSafeDialContext, logf: logf,
}
p.srv = &http.Server{Handler: p, ReadHeaderTimeout: 30 * time.Second}
p.wg.Add(1)
go func() {
defer p.wg.Done()
_ = p.srv.Serve(ln)
}()
return p, nil
}
// URL is the proxy address for the plugin's environment.
func (p *egressProxy) URL() string { return "http://" + p.ln.Addr().String() }
func (p *egressProxy) Close() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = p.srv.Shutdown(ctx)
p.wg.Wait()
}
func (p *egressProxy) deny(w http.ResponseWriter, host string) {
p.logf("[plugin] %s: egress to %s refused (not in permissions.egress)", p.pluginID, host)
http.Error(w, fmt.Sprintf("egress to %s is not permitted for plugin %s", host, p.pluginID), http.StatusForbidden)
}
func (p *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodConnect {
p.tunnel(w, r)
return
}
if r.URL.Host == "" {
http.Error(w, "not a proxy request", http.StatusBadRequest)
return
}
host := r.URL.Hostname()
if !p.policy.allows(host) {
p.deny(w, host)
return
}
out := r.Clone(r.Context())
out.RequestURI = ""
out.Header.Del("Proxy-Connection")
out.Header.Del("Proxy-Authorization")
tr := &http.Transport{DialContext: p.dial, ResponseHeaderTimeout: 2 * time.Minute}
defer tr.CloseIdleConnections()
resp, err := tr.RoundTrip(out)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
defer func() { _ = resp.Body.Close() }()
for k, vs := range resp.Header {
for _, v := range vs {
w.Header().Add(k, v)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
// tunnel handles CONNECT, which HTTPS traffic uses: the proxy sees only the
// host and port, which is what the policy needs.
func (p *egressProxy) tunnel(w http.ResponseWriter, r *http.Request) {
host, _, err := net.SplitHostPort(r.Host)
if err != nil {
http.Error(w, "bad CONNECT target", http.StatusBadRequest)
return
}
if !p.policy.allows(host) {
p.deny(w, host)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
upstream, err := p.dial(ctx, "tcp", r.Host)
cancel()
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
hj, ok := w.(http.Hijacker)
if !ok {
_ = upstream.Close()
http.Error(w, "hijacking unsupported", http.StatusInternalServerError)
return
}
client, buf, err := hj.Hijack()
if err != nil {
_ = upstream.Close()
return
}
if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
_ = upstream.Close()
_ = client.Close()
return
}
if buf.Reader.Buffered() > 0 {
pending := make([]byte, buf.Reader.Buffered())
_, _ = buf.Read(pending)
_, _ = upstream.Write(pending)
}
go func() {
_, _ = io.Copy(upstream, client)
_ = upstream.Close()
}()
_, _ = io.Copy(client, upstream)
_ = client.Close()
}

View File

@ -0,0 +1,44 @@
// Command echoplugin is a host plugin for the host package's tests. Its
// behaviour is driven by the query it is asked and by a "version" file next
// to it, so a test can install a package that lies about its version.
package main
import (
"context"
"log"
"os"
"sort"
"strconv"
"strings"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
func main() {
version := os.Getenv(pluginapi.EnvPluginVersion)
if b, err := os.ReadFile("version"); err == nil {
version = strings.TrimSpace(string(b))
}
p := pluginsdk.New(pluginsdk.Info{ID: os.Getenv(pluginapi.EnvPluginID), Version: version})
p.WebSearch("echo", pluginsdk.WebSearchFunc(func(_ context.Context, _ *pluginsdk.Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
switch in.Query {
case "crash":
os.Exit(3)
case "env":
var keys []string
for _, kv := range os.Environ() {
k, _, _ := strings.Cut(kv, "=")
keys = append(keys, k)
}
sort.Strings(keys)
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: strings.Join(keys, ","), URL: "env"}}}, nil
case "pid":
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: strconv.Itoa(os.Getpid()), URL: "pid"}}}, nil
}
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: in.Query, URL: "echo"}}}, nil
}))
if err := p.Serve(); err != nil {
log.Fatal(err)
}
}

View File

@ -12,7 +12,10 @@ import (
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"path"
goruntime "runtime"
"sort" "sort"
"strings"
"time" "time"
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
@ -41,8 +44,29 @@ func invalid(format string, args ...any) error {
} }
// supportedRuntimes are the runtimes an installed package may use today. // supportedRuntimes are the runtimes an installed package may use today.
// Code plugins arrive with the plugin host. var supportedRuntimes = map[manifest.RuntimeType]bool{
var supportedRuntimes = map[manifest.RuntimeType]bool{manifest.RuntimeDeclarative: true} manifest.RuntimeDeclarative: true,
manifest.RuntimeHost: true,
}
// checkHostRuntime makes sure this server can run a host plugin: a binary
// built for its OS and architecture. Other kinds need a host image that
// carries their interpreter.
func checkHostRuntime(p *pkg.Package) error {
rt := p.Manifest.Runtime
if rt.Kind != "binary" {
return invalid("runtime.kind %q is not supported yet; host plugins must be binaries", rt.Kind)
}
entry := strings.NewReplacer("{os}", goruntime.GOOS, "{arch}", goruntime.GOARCH).Replace(rt.Entry)
if goruntime.GOOS == "windows" && path.Ext(entry) == "" {
entry += ".exe"
}
if _, ok := p.ReadFile(entry); !ok {
return invalid("the package has no build for this server (%s/%s): %s is missing",
goruntime.GOOS, goruntime.GOARCH, entry)
}
return nil
}
// Syncer is the node's reconciler as the installer uses it. // Syncer is the node's reconciler as the installer uses it.
type Syncer interface { type Syncer interface {
@ -152,9 +176,14 @@ func (s *Service) open(data []byte) (*pkg.Package, error) {
return nil, &InvalidError{Err: err} return nil, &InvalidError{Err: err}
} }
if !supportedRuntimes[p.Manifest.Runtime.Type] { if !supportedRuntimes[p.Manifest.Runtime.Type] {
return nil, invalid("runtime %q is not supported yet; only declarative plugins can be installed", return nil, invalid("runtime %q is not supported yet; declarative and host plugins can be installed",
p.Manifest.Runtime.Type) p.Manifest.Runtime.Type)
} }
if p.Manifest.Runtime.Type == manifest.RuntimeHost {
if err := checkHostRuntime(p); err != nil {
return nil, err
}
}
if err := p.Manifest.CheckEngines(s.hostVersion); err != nil { if err := p.Manifest.CheckEngines(s.hostVersion); err != nil {
return nil, &InvalidError{Err: err} return nil, &InvalidError{Err: err}
} }

View File

@ -3,6 +3,7 @@ package install
import ( import (
"context" "context"
"errors" "errors"
goruntime "runtime"
"strings" "strings"
"testing" "testing"
@ -127,3 +128,36 @@ func isInvalid(err error) bool {
var ie *InvalidError var ie *InvalidError
return errors.As(err, &ie) return errors.As(err, &ie)
} }
func hostPackage(t *testing.T, kind, binPath string) []byte {
return plugintest.Zip(t, map[string]string{
"plugin.yaml": "schemaVersion: 1\nid: acme.search\nversion: 1.0.0\napiVersion: weknora.plugin/v1\n" +
"name: { en-US: ACME Search }\npublisher: { id: acme }\n" +
"runtime: { type: host, kind: " + kind + ", entry: \"bin/{os}-{arch}/search\" }\n" +
"permissions: { egress: [api.acme.example] }\n" +
"contributes:\n webSearch:\n - { id: search, name: ACME Search }\n",
binPath: "binary",
})
}
func TestInstallHostPlugins(t *testing.T) {
ctx := context.Background()
s, _, _, _ := newService(t)
name := "search"
if goruntime.GOOS == "windows" {
name += ".exe"
}
here := "bin/" + goruntime.GOOS + "-" + goruntime.GOARCH + "/" + name
p, err := s.Inspect(ctx, hostPackage(t, "binary", here))
if err != nil || p.Manifest.Permissions.Egress[0] != "api.acme.example" {
t.Fatalf("Inspect = %+v, %v", p, err)
}
if _, err := s.Inspect(ctx, hostPackage(t, "binary", "bin/plan9-mips/search")); !isInvalid(err) ||
!strings.Contains(err.Error(), "no build for this server") {
t.Fatalf("want a missing build error, got %v", err)
}
if _, err := s.Inspect(ctx, hostPackage(t, "python", here)); !isInvalid(err) ||
!strings.Contains(err.Error(), "must be binaries") {
t.Fatalf("want an unsupported kind error, got %v", err)
}
}

View File

@ -57,15 +57,15 @@ type Manifest struct {
// Builtin marks a plugin compiled into WeKnora. It cannot be set from a // Builtin marks a plugin compiled into WeKnora. It cannot be set from a
// manifest file. // manifest file.
Builtin bool `json:"builtin,omitempty" yaml:"-"` Builtin bool `json:"builtin,omitempty" yaml:"-"`
// Required marks a builtin a tenant may not disable (the core agent // Required marks a builtin a tenant may not disable (the core agent
// tools, the default parser). // tools, the default parser).
Required bool `json:"required,omitempty" yaml:"-"` Required bool `json:"required,omitempty" yaml:"-"`
Runtime Runtime `json:"runtime" yaml:"runtime"` Runtime Runtime `json:"runtime" yaml:"runtime"`
Permissions Permissions `json:"permissions,omitzero" yaml:"permissions"` Permissions Permissions `json:"permissions,omitzero" yaml:"permissions"`
Config ConfigSchemas `json:"config,omitzero" yaml:"config"` Config ConfigSchemas `json:"config,omitzero" yaml:"config"`
Contributes Contributions `json:"contributes" yaml:"contributes"` Contributes Contributions `json:"contributes" yaml:"contributes"`
} }
// Engines constrains which WeKnora versions a plugin runs on. // Engines constrains which WeKnora versions a plugin runs on.
@ -84,12 +84,12 @@ type Publisher struct {
// Runtime says how a plugin's code runs. // Runtime says how a plugin's code runs.
type Runtime struct { type Runtime struct {
Type RuntimeType `json:"type" yaml:"type"` Type RuntimeType `json:"type" yaml:"type"`
// Kind is the process runtime for host plugins: binary, python or node. // Kind is the process runtime for host plugins: binary, python or node.
Kind string `json:"kind,omitempty" yaml:"kind"` Kind string `json:"kind,omitempty" yaml:"kind"`
// Entry is the command inside the package, with {os} and {arch} // Entry is the command inside the package, with {os} and {arch}
// placeholders for binaries. // placeholders for binaries.
Entry string `json:"entry,omitempty" yaml:"entry"` Entry string `json:"entry,omitempty" yaml:"entry"`
// Singleton allows only one live instance across the cluster (long-lived // Singleton allows only one live instance across the cluster (long-lived
// connections such as an IM WebSocket). // connections such as an IM WebSocket).
Singleton bool `json:"singleton,omitempty" yaml:"singleton"` Singleton bool `json:"singleton,omitempty" yaml:"singleton"`
@ -106,17 +106,17 @@ type Resources struct {
// install time. // install time.
type Permissions struct { type Permissions struct {
// Egress lists host patterns the plugin may reach ("*.atlassian.net"). // Egress lists host patterns the plugin may reach ("*.atlassian.net").
Egress []string `json:"egress,omitempty" yaml:"egress"` Egress []string `json:"egress,omitempty" yaml:"egress"`
// HostAPI lists WeKnora API scopes the plugin may call back with. // HostAPI lists WeKnora API scopes the plugin may call back with.
HostAPI []string `json:"hostApi,omitempty" yaml:"hostApi"` HostAPI []string `json:"hostApi,omitempty" yaml:"hostApi"`
// Events lists asynchronous events the plugin subscribes to. // Events lists asynchronous events the plugin subscribes to.
Events []string `json:"events,omitempty" yaml:"events"` Events []string `json:"events,omitempty" yaml:"events"`
} }
// ConfigSchemas points at the JSON Schemas of plugin-level configuration. // ConfigSchemas points at the JSON Schemas of plugin-level configuration.
type ConfigSchemas struct { type ConfigSchemas struct {
System string `json:"system,omitempty" yaml:"system"` System string `json:"system,omitempty" yaml:"system"`
Tenant string `json:"tenant,omitempty" yaml:"tenant"` Tenant string `json:"tenant,omitempty" yaml:"tenant"`
// SystemSchema and TenantSchema are the schema files' contents as JSON, // SystemSchema and TenantSchema are the schema files' contents as JSON,
// filled in when a package is opened. // filled in when a package is opened.
SystemSchema json.RawMessage `json:"systemSchema,omitempty" yaml:"-"` SystemSchema json.RawMessage `json:"systemSchema,omitempty" yaml:"-"`
@ -130,40 +130,40 @@ type Contributions map[Point][]Contribution
// such as one connector. // such as one connector.
type Contribution struct { type Contribution struct {
// ID is local to the plugin; QualifiedID adds the plugin ID. // ID is local to the plugin; QualifiedID adds the plugin ID.
ID string `json:"id" yaml:"id"` ID string `json:"id" yaml:"id"`
Name LocalizedText `json:"name" yaml:"name"` Name LocalizedText `json:"name" yaml:"name"`
Description LocalizedText `json:"description,omitzero" yaml:"description"` Description LocalizedText `json:"description,omitzero" yaml:"description"`
Icon string `json:"icon,omitempty" yaml:"icon"` Icon string `json:"icon,omitempty" yaml:"icon"`
// Aliases are the short names builtins were stored under before plugins // Aliases are the short names builtins were stored under before plugins
// existed ("feishu"). Rows keep them, so lookups must keep resolving // existed ("feishu"). Rows keep them, so lookups must keep resolving
// them. Only builtins may declare aliases. // them. Only builtins may declare aliases.
Aliases []string `json:"aliases,omitempty" yaml:"-"` Aliases []string `json:"aliases,omitempty" yaml:"-"`
Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities"` Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities"`
// Order sorts contributions of one point in the UI (lower first); ties // Order sorts contributions of one point in the UI (lower first); ties
// keep registration order. // keep registration order.
Order int `json:"order,omitempty" yaml:"order"` Order int `json:"order,omitempty" yaml:"order"`
// InstanceSchema points at the JSON Schema of one instance's // InstanceSchema points at the JSON Schema of one instance's
// configuration (one data source, one IM channel). // configuration (one data source, one IM channel).
InstanceSchema string `json:"instanceSchema,omitempty" yaml:"instanceSchema"` InstanceSchema string `json:"instanceSchema,omitempty" yaml:"instanceSchema"`
// Path is a file or directory inside the package: the skill directory // Path is a file or directory inside the package: the skill directory
// (skills) or the vendor definition JSON (modelVendors). // (skills) or the vendor definition JSON (modelVendors).
Path string `json:"path,omitempty" yaml:"path"` Path string `json:"path,omitempty" yaml:"path"`
// MCP describes a remote MCP server (mcpServers). // MCP describes a remote MCP server (mcpServers).
MCP *MCPServer `json:"mcp,omitempty" yaml:"mcp"` MCP *MCPServer `json:"mcp,omitempty" yaml:"mcp"`
// Extra carries point-specific metadata the generic fields do not cover. // Extra carries point-specific metadata the generic fields do not cover.
Extra map[string]any `json:"extra,omitempty" yaml:"extra"` Extra map[string]any `json:"extra,omitempty" yaml:"extra"`
} }
// MCPServer is a remote MCP server a plugin contributes. // MCPServer is a remote MCP server a plugin contributes.
type MCPServer struct { type MCPServer struct {
URL string `json:"url" yaml:"url"` URL string `json:"url" yaml:"url"`
// Transport is "sse" or "http-streamable" (the default). // Transport is "sse" or "http-streamable" (the default).
Transport string `json:"transport,omitempty" yaml:"transport"` Transport string `json:"transport,omitempty" yaml:"transport"`
// Headers are sent on every request. A value may reference the // Headers are sent on every request. A value may reference the
// workspace's plugin configuration as ${config.<key>} (how a plugin asks // workspace's plugin configuration as ${config.<key>} (how a plugin asks
// each workspace for its own API key) or the platform's as // each workspace for its own API key) or the platform's as
// ${system.<key>}. The URL is fixed: configuration cannot redirect it. // ${system.<key>}. The URL is fixed: configuration cannot redirect it.
Headers map[string]string `json:"headers,omitempty" yaml:"headers"` Headers map[string]string `json:"headers,omitempty" yaml:"headers"`
} }
// MCP transports a plugin may declare. // MCP transports a plugin may declare.
@ -238,6 +238,7 @@ func (m *Manifest) Validate() error {
} }
} }
m.validateRuntime(add) m.validateRuntime(add)
m.validatePermissions(add)
m.validateContributions(add) m.validateContributions(add)
return errors.Join(errs...) return errors.Join(errs...)
} }
@ -248,8 +249,11 @@ func (m *Manifest) validateRuntime(add func(string, ...any)) {
if !m.Builtin { if !m.Builtin {
add("runtime.type %q is reserved for builtin plugins", RuntimeBuiltin) add("runtime.type %q is reserved for builtin plugins", RuntimeBuiltin)
} }
case RuntimeDeclarative, RuntimeRemote, RuntimeKubernetes: case RuntimeDeclarative:
case RuntimeRemote, RuntimeKubernetes:
m.validateAPIVersion(add)
case RuntimeHost: case RuntimeHost:
m.validateAPIVersion(add)
switch m.Runtime.Kind { switch m.Runtime.Kind {
case "binary", "python", "node": case "binary", "python", "node":
default: default:
@ -263,6 +267,42 @@ func (m *Manifest) validateRuntime(add func(string, ...any)) {
} }
} }
// ExtensionAPIVersion is the extension protocol this build speaks
// (pluginsdk/pluginapi.APIVersion).
const ExtensionAPIVersion = "weknora.plugin/v1"
func (m *Manifest) validateAPIVersion(add func(string, ...any)) {
if m.Builtin {
return
}
if m.APIVersion != ExtensionAPIVersion {
add("apiVersion must be %q for code plugins, got %q", ExtensionAPIVersion, m.APIVersion)
}
}
// egressPattern is a host name, optionally with a leading "*." wildcard.
var egressPattern = regexp.MustCompile(`^(\*\.)?([a-z0-9]([a-z0-9-]*[a-z0-9])?)(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$`)
// EgressAnyHost in permissions.egress grants any public host (a feed reader
// follows links anywhere). Private addresses stay unreachable either way.
const EgressAnyHost = "*"
func (m *Manifest) validatePermissions(add func(string, ...any)) {
for _, e := range m.Permissions.Egress {
if e != EgressAnyHost && !egressPattern.MatchString(strings.ToLower(e)) {
add(
"permissions.egress %q must be a host name (api.example.com), a wildcard (*.example.com) or *",
e,
)
}
}
}
// storedTypeIDLimit is how long a qualified contribution ID may be where
// instances store it in a varchar(50) column (data_sources.type,
// web_search_providers.provider, knowledges.channel).
var storedTypeIDLimit = map[Point]int{PointConnectors: 50, PointWebSearch: 50}
func (m *Manifest) validateContributions(add func(string, ...any)) { func (m *Manifest) validateContributions(add func(string, ...any)) {
if len(m.Contributes) == 0 { if len(m.Contributes) == 0 {
add("contributes must declare at least one contribution") add("contributes must declare at least one contribution")
@ -297,6 +337,10 @@ func (m *Manifest) validateContributions(add func(string, ...any)) {
if c.Name.IsZero() { if c.Name.IsZero() {
add("%s.name is required", where) add("%s.name is required", where)
} }
if limit, ok := storedTypeIDLimit[point]; ok && !m.Builtin && len(QualifiedID(m.ID, c.ID)) > limit {
add("%s: %q is longer than the %d characters instances store their type in",
where, QualifiedID(m.ID, c.ID), limit)
}
if len(c.Aliases) > 0 && !m.Builtin { if len(c.Aliases) > 0 && !m.Builtin {
add("%s.aliases may only be declared by builtin plugins", where) add("%s.aliases may only be declared by builtin plugins", where)
} }

View File

@ -76,7 +76,8 @@ func TestValidateReportsEveryProblem(t *testing.T) {
Version: "1.0", Version: "1.0",
Runtime: Runtime{Type: RuntimeHost}, Runtime: Runtime{Type: RuntimeHost},
Contributes: Contributions{ Contributes: Contributions{
"widgets": {{ID: "x", Name: Text("X", nil)}}, "widgets": {{ID: "x", Name: Text("X", nil)}},
PointIMChannels: {{ID: "im", Name: Text("IM", nil)}},
PointConnectors: { PointConnectors: {
{ID: "dup", Name: Text("A", nil), Aliases: []string{"old"}}, {ID: "dup", Name: Text("A", nil), Aliases: []string{"old"}},
{ID: "dup"}, {ID: "dup"},
@ -94,8 +95,9 @@ func TestValidateReportsEveryProblem(t *testing.T) {
"name is required", "name is required",
"runtime.kind", "runtime.kind",
"runtime.entry", "runtime.entry",
"apiVersion must be",
"contributes.widgets is not a known extension point", "contributes.widgets is not a known extension point",
"is not open to third-party plugins yet", "contributes.imChannels is not open to third-party plugins yet",
`"dup" is declared twice`, `"dup" is declared twice`,
"contributes.connectors[1].name is required", "contributes.connectors[1].name is required",
"aliases may only be declared by builtin plugins", "aliases may only be declared by builtin plugins",
@ -184,3 +186,35 @@ func TestContributionOmitsEmptyDescription(t *testing.T) {
t.Fatalf("empty description should be omitted: %s", data) t.Fatalf("empty description should be omitted: %s", data)
} }
} }
func TestEgressPatterns(t *testing.T) {
base := func(egress ...string) *Manifest {
return &Manifest{
SchemaVersion: SchemaVersion, ID: "acme.x", Version: "1.0.0", APIVersion: ExtensionAPIVersion,
Name: Text("X", nil), Publisher: Publisher{ID: "acme"},
Runtime: Runtime{Type: RuntimeHost, Kind: "binary", Entry: "bin/x"},
Permissions: Permissions{Egress: egress},
Contributes: Contributions{PointWebSearch: {{ID: "x", Name: Text("X", nil)}}},
}
}
if err := base("api.example.com", "*.atlassian.net", "*").Validate(); err != nil {
t.Fatalf("valid egress rejected: %v", err)
}
for _, bad := range []string{"**", "http://x.com", "x.com:443", "*.*.com", "10.0.0.0/8"} {
if err := base(bad).Validate(); err == nil || !strings.Contains(err.Error(), "permissions.egress") {
t.Errorf("egress %q: got %v", bad, err)
}
}
}
func TestStoredTypeIDLength(t *testing.T) {
m := &Manifest{
SchemaVersion: SchemaVersion, ID: "acme.a-rather-long-plugin-name", Version: "1.0.0",
APIVersion: ExtensionAPIVersion, Name: Text("X", nil), Publisher: Publisher{ID: "acme"},
Runtime: Runtime{Type: RuntimeHost, Kind: "binary", Entry: "bin/x"},
Contributes: Contributions{PointConnectors: {{ID: "and-an-even-longer-connector-id", Name: Text("X", nil)}}},
}
if err := m.Validate(); err == nil || !strings.Contains(err.Error(), "longer than the 50 characters") {
t.Fatalf("want a length error, got %v", err)
}
}

View File

@ -36,9 +36,9 @@ type PointInfo struct {
// infrastructure owned by WeKnora itself, not extension points. // infrastructure owned by WeKnora itself, not extension points.
var points = []PointInfo{ var points = []PointInfo{
{Point: PointModelVendors, ThirdParty: true, Declarative: true}, {Point: PointModelVendors, ThirdParty: true, Declarative: true},
{Point: PointConnectors}, {Point: PointConnectors, ThirdParty: true},
{Point: PointIMChannels}, {Point: PointIMChannels},
{Point: PointWebSearch}, {Point: PointWebSearch, ThirdParty: true},
{Point: PointTools}, {Point: PointTools},
{Point: PointParsers}, {Point: PointParsers},
{Point: PointSkills, ThirdParty: true, Declarative: true}, {Point: PointSkills, ThirdParty: true, Declarative: true},

View File

@ -117,7 +117,7 @@ func TestOpenRejectsBadPackages(t *testing.T) {
"config/tenant.yaml": tenantSchema, "config/tenant.yaml": tenantSchema,
"skills/triage/SKILL.md": "x", "skills/triage/SKILL.md": "x",
}, },
want: "not open to third-party plugins", want: "contributes.connectors needs code",
}, },
} }
for name, c := range cases { for name, c := range cases {

View File

@ -106,23 +106,29 @@ func (r *Reconciler) NodeStatuses(ctx context.Context, pluginID string) ([]NodeS
return out, nil return out, nil
} }
// Driver returns the driver for declarative plugins. They have no code to // Driver returns the status driver for plugins the reconciler loads on every
// run, so an instance is a node that loaded the plugin. // node (declarative and host): an instance is a node that loaded the plugin,
func (r *Reconciler) Driver() driver.Driver { return declarativeDriver{r} } // with the runtime health the node reported.
func (r *Reconciler) Driver(rt manifest.RuntimeType) driver.Driver { return nodeDriver{r: r, rt: rt} }
type declarativeDriver struct{ r *Reconciler } type nodeDriver struct {
r *Reconciler
func (declarativeDriver) Type() manifest.RuntimeType { return manifest.RuntimeDeclarative } rt manifest.RuntimeType
// Ensure and Remove are the reconciler's job for declarative plugins.
func (declarativeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
func (declarativeDriver) Remove(context.Context, string, string) error { return nil }
func (declarativeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
return driver.Endpoint{}, fmt.Errorf("declarative plugin %s has no endpoint", pluginID)
} }
func (d declarativeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) { func (d nodeDriver) Type() manifest.RuntimeType { return d.rt }
// Ensure and Remove are the reconciler's job.
func (nodeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
func (nodeDriver) Remove(context.Context, string, string) error { return nil }
// Resolve is not how these plugins are reached: declarative ones have no
// endpoint and host ones are reached through the node's host manager.
func (d nodeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
return driver.Endpoint{}, fmt.Errorf("%s plugin %s has no shared endpoint", d.rt, pluginID)
}
func (d nodeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) {
nodes, err := d.r.NodeStatuses(ctx, pluginID) nodes, err := d.r.NodeStatuses(ctx, pluginID)
if err != nil { if err != nil {
return nil, err return nil, err
@ -130,7 +136,11 @@ func (d declarativeDriver) Status(ctx context.Context, pluginID string) ([]drive
out := make([]driver.InstanceStatus, 0, len(nodes)) out := make([]driver.InstanceStatus, 0, len(nodes))
for _, n := range nodes { for _, n := range nodes {
state := driver.StateReady state := driver.StateReady
if n.State != StateReady { switch n.State {
case StateReady:
case StateDegraded:
state = driver.StateDegraded
default:
state = driver.StateStopped state = driver.StateStopped
} }
out = append(out, driver.InstanceStatus{ out = append(out, driver.InstanceStatus{

View File

@ -8,6 +8,7 @@ import (
"github.com/alicebob/miniredis/v2" "github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/plugintest" "github.com/Tencent/WeKnora/internal/plugin/plugintest"
"github.com/Tencent/WeKnora/internal/plugin/registry" "github.com/Tencent/WeKnora/internal/plugin/registry"
"github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types"
@ -45,7 +46,7 @@ func TestNodesConvergeThroughRedis(t *testing.T) {
nodes, err := a.NodeStatuses(ctx, "acme.kit") nodes, err := a.NodeStatuses(ctx, "acme.kit")
return err == nil && len(nodes) == 2 return err == nil && len(nodes) == 2
}) })
instances, err := b.Driver().Status(ctx, "acme.kit") instances, err := b.Driver(manifest.RuntimeDeclarative).Status(ctx, "acme.kit")
if err != nil || len(instances) != 2 || instances[0].State != "ready" { if err != nil || len(instances) != 2 || instances[0].State != "ready" {
t.Fatalf("instances = %+v, %v", instances, err) t.Fatalf("instances = %+v, %v", instances, err)
} }

View File

@ -49,6 +49,14 @@ type Activator interface {
Deactivate(ctx context.Context, pluginID string) error Deactivate(ctx context.Context, pluginID string) error
} }
// InPlaceActivator swaps an upgraded plugin itself inside Activate (the host
// starts the new process before stopping the old one), so it is not
// deactivated first.
type InPlaceActivator interface {
Activator
ActivatesInPlace()
}
// Status is how a plugin fares on this node. // Status is how a plugin fares on this node.
type Status struct { type Status struct {
Version string `json:"version"` Version string `json:"version"`
@ -61,6 +69,9 @@ type Status struct {
const ( const (
StateReady = "ready" StateReady = "ready"
StateFailed = "failed" StateFailed = "failed"
// StateDegraded: loaded, but its process crashed or fails health checks
// and is being restarted.
StateDegraded = "degraded"
) )
// DefaultInterval is how often a node reconciles without being told to. // DefaultInterval is how often a node reconciles without being told to.
@ -208,7 +219,8 @@ func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) erro
l := &Loaded{Manifest: p.Manifest, Package: p, Dir: dir} l := &Loaded{Manifest: p.Manifest, Package: p, Dir: dir}
var errs []error var errs []error
for _, a := range r.activators { for _, a := range r.activators {
if _, had := r.loaded[row.ID]; had { _, inPlace := a.(InPlaceActivator)
if _, had := r.loaded[row.ID]; had && !inPlace {
if err := a.Deactivate(ctx, row.ID); err != nil { if err := a.Deactivate(ctx, row.ID); err != nil {
errs = append(errs, fmt.Errorf("%s: deactivate previous version: %w", a.Name(), err)) errs = append(errs, fmt.Errorf("%s: deactivate previous version: %w", a.Name(), err))
} }
@ -228,7 +240,9 @@ func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) erro
} }
func (r *Reconciler) unload(ctx context.Context, id string) { func (r *Reconciler) unload(ctx context.Context, id string) {
for _, a := range r.activators { // Reverse order: routes go before the processes they route to.
for i := len(r.activators) - 1; i >= 0; i-- {
a := r.activators[i]
if err := a.Deactivate(ctx, id); err != nil { if err := a.Deactivate(ctx, id); err != nil {
logger.Warnf(ctx, "[plugin] %s: deactivate %s: %v", a.Name(), id, err) logger.Warnf(ctx, "[plugin] %s: deactivate %s: %v", a.Name(), id, err)
} }
@ -286,6 +300,28 @@ func (r *Reconciler) setStatus(id string, s Status) {
r.statusMu.Unlock() r.statusMu.Unlock()
} }
// ReportRuntime records a loaded plugin's runtime health (a host process that
// crashed and is restarting, or recovered) and publishes it at once.
func (r *Reconciler) ReportRuntime(pluginID string, healthy bool, err error) {
r.statusMu.Lock()
s, ok := r.status[pluginID]
if !ok || s.State == StateFailed {
r.statusMu.Unlock()
return
}
s.State, s.Error = StateReady, ""
if !healthy {
s.State = StateDegraded
if err != nil {
s.Error = err.Error()
}
}
s.UpdatedAt = time.Now()
r.status[pluginID] = s
r.statusMu.Unlock()
r.publishStatuses(context.Background())
}
// Status reports how one plugin fares on this node. // Status reports how one plugin fares on this node.
func (r *Reconciler) Status(pluginID string) (Status, bool) { func (r *Reconciler) Status(pluginID string) (Status, bool) {
r.statusMu.RLock() r.statusMu.RLock()

View File

@ -41,7 +41,11 @@ type DataSourceService interface {
// ValidateCredentials tests connectivity using raw credentials without persisting anything. // ValidateCredentials tests connectivity using raw credentials without persisting anything.
// This is used by the frontend "Test Connection" button before creating a data source. // This is used by the frontend "Test Connection" button before creating a data source.
ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error // ValidateCredentials tests unsaved credentials (and settings, which
// some connectors need to reach the source) against the connector.
ValidateCredentials(
ctx context.Context, connectorType string, credentials, settings map[string]interface{},
) error
// ListAvailableResources lists resources available for sync in the external system. // ListAvailableResources lists resources available for sync in the external system.
// parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children. // parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children.

View File

@ -134,6 +134,13 @@ type WebSearchProviderTypeInfo struct {
ID string `json:"id"` ID string `json:"id"`
// Human-readable name // Human-readable name
Name string `json:"name"` Name string `json:"name"`
// Names localizes Name for plugin provider types, keyed by locale.
Names map[string]string `json:"names,omitempty"`
// Icon is an image (data: URI) for plugin provider types; builtin types
// use the frontend's bundled logos.
Icon string `json:"icon,omitempty"`
// PluginID names the installed plugin providing the type.
PluginID string `json:"plugin_id,omitempty"`
// Whether the provider requires an API key // Whether the provider requires an API key
RequiresAPIKey bool `json:"requires_api_key"` RequiresAPIKey bool `json:"requires_api_key"`
// Whether the provider accepts an optional API key (keyless by default, but a // Whether the provider accepts an optional API key (keyless by default, but a

120
pluginsdk/README.md 100644
View File

@ -0,0 +1,120 @@
# WeKnora plugin SDK (Go)
Build code plugins for WeKnora: web search providers and data source
connectors that run as their own process. The module has no dependencies.
| Package | What it is |
| --- | --- |
| `pluginapi` | The extension protocol v1: envelope, errors, stream events, messages. `openapi.yaml` describes it for other languages. |
| `pluginsdk` | Write a plugin: register contributions, then `Serve`. |
| `client` | Call a plugin, as WeKnora does. |
| `conformance`, `cmd/weknora-plugin-conformance` | Check that a plugin speaks the protocol. |
## A connector in 30 lines
```go
package main
import (
"context"
"log"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
type notes struct{}
func (notes) Validate(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
if cfg.Credentials["token"] == nil {
return pluginapi.InvalidConfig("token is required", map[string]string{"credentials.token": "required"})
}
return nil
}
func (notes) ListResources(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig, parent string) ([]pluginapi.Resource, error) {
return []pluginapi.Resource{{ExternalID: "inbox", Name: "Inbox"}}, nil
}
func (notes) Fetch(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig, in pluginapi.FetchInput, s *pluginsdk.Stream) (*pluginapi.Cursor, error) {
if err := s.Item(pluginapi.FetchedItem{ExternalID: "n1", Title: "First note", Content: []byte("# Hello")}); err != nil {
return nil, err
}
return &pluginapi.Cursor{State: map[string]any{"after": "n1"}}, nil
}
func main() {
p := pluginsdk.New(pluginsdk.Info{ID: "acme.notes", Version: "1.0.0"})
p.Connector("notes", notes{})
log.Fatal(p.Serve())
}
```
What the SDK does for you:
- **Handshake.** Started by WeKnora's plugin host, the plugin listens on the
socket it is given and accepts only the host's token.
- **Remote mode.** Run on its own, it listens on `WEKNORA_PLUGIN_ADDR` and
verifies signed requests with `WEKNORA_PLUGIN_SECRET`.
- **Per call.** Each call's deadline reaches your context. Returned
`pluginapi` errors keep their code; anything else, and any panic, becomes
`internal`.
Every call carries the tenant and the configuration, secrets already
decrypted, in three separate scopes:
- `call.Config.System`: set by the system administrator.
- `call.Config.Tenant`: set per workspace.
- the instance: `cfg`, or `call.DecodeInstance`.
Keep plugins stateless and never cache a tenant's credentials across calls.
## Packaging
A package is a zip (`.wkp`) with `plugin.yaml` at its root:
```yaml
schemaVersion: 1
id: acme.notes
version: 1.0.0
apiVersion: weknora.plugin/v1
name: { en-US: ACME Notes }
publisher: { id: acme }
runtime: { type: host, kind: binary, entry: "bin/{os}-{arch}/notes" }
permissions:
egress: [api.acme.example] # hosts the plugin may reach; "*" = any public host
contributes:
connectors:
- id: notes
name: { en-US: ACME Notes }
instanceSchema: schemas/notes.yaml # fields with x-group: settings are settings, the rest credentials
```
A few rules the manifest and host enforce:
- **Binaries.** Build one per platform into `bin/<os>-<arch>/`; WeKnora
runs the build for its own OS and architecture.
- **Environment.** The process gets no environment from WeKnora beyond the
`WEKNORA_PLUGIN_*` variables.
- **Outbound traffic.** It goes through the host's egress proxy, which
forwards only to the hosts in `permissions.egress` and never to private
addresses.
- **IDs.** Contribution IDs are qualified with the plugin ID
(`acme.notes/notes`); for connectors and web search that must stay within
50 characters.
See `examples/plugins/rss` for a complete plugin and its `package.sh`.
## Testing
```bash
go run github.com/Tencent/WeKnora/pluginsdk/cmd/weknora-plugin-conformance -url http://localhost:8080 -secret $SECRET
```
In Go tests, run `conformance.Run` against `httptest.NewServer(p.Handler())`.
## Installing
In WeKnora: **System administration → Plugin management → Install plugin**.
Upload the `.wkp` or give its URL, review what it adds and reaches, and
install. Each workspace then enables it under **Settings → Plugins**.

View File

@ -0,0 +1,259 @@
// Package client calls a WeKnora code plugin over the extension protocol. It
// is what WeKnora itself uses, and what the conformance suite tests plugins
// with.
package client
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Auth adds credentials to a request.
type Auth interface {
Apply(req *http.Request, body []byte) error
}
// Bearer authenticates as the plugin host.
type Bearer string
// Apply implements Auth.
func (b Bearer) Apply(req *http.Request, _ []byte) error {
req.Header.Set("Authorization", "Bearer "+string(b))
return nil
}
// Signed authenticates to a remote plugin with its shared secret.
type Signed []byte
// Apply implements Auth.
func (s Signed) Apply(req *http.Request, body []byte) error {
ts := time.Now().Unix()
req.Header.Set(pluginapi.TimestampHeader, strconv.FormatInt(ts, 10))
req.Header.Set(pluginapi.SignatureHeader, pluginapi.Sign(s, ts, body))
return nil
}
// TransportError is a failure without a protocol answer: the plugin was
// unreachable, answered something that is not the protocol, or broke off a
// stream. It unwraps to a *pluginapi.Error (unavailable or internal) so
// callers can treat every failure alike.
type TransportError struct {
Err *pluginapi.Error
}
func (e *TransportError) Error() string { return e.Err.Error() }
func (e *TransportError) Unwrap() error { return e.Err }
func transport(code pluginapi.ErrorCode, format string, args ...any) error {
return &TransportError{Err: &pluginapi.Error{
Code: code, Message: fmt.Sprintf(format, args...), Retryable: code == pluginapi.CodeUnavailable,
}}
}
// Client calls one plugin process or service.
type Client struct {
base string
http *http.Client
auth Auth
}
// ForHandshake reaches a host plugin at the address its handshake names.
// The returned client has no overall timeout: calls are bounded by their
// context.
func ForHandshake(h pluginapi.Handshake, auth Auth) *Client {
if h.Network == "unix" {
tr := &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", h.Address)
},
MaxIdleConnsPerHost: 16,
IdleConnTimeout: 90 * time.Second,
}
return &Client{base: "http://plugin", http: &http.Client{Transport: tr}, auth: auth}
}
return New("http://"+h.Address, nil, auth)
}
// New reaches a plugin at an HTTP(S) base URL. A nil httpClient uses a
// client without an overall timeout.
func New(baseURL string, httpClient *http.Client, auth Auth) *Client {
if httpClient == nil {
httpClient = &http.Client{}
}
return &Client{base: strings.TrimSuffix(baseURL, "/"), http: httpClient, auth: auth}
}
// Close releases idle connections.
func (c *Client) Close() { c.http.CloseIdleConnections() }
func (c *Client) do(ctx context.Context, method, path string, body []byte, requestID string) (*http.Response, error) {
var rd io.Reader
if body != nil {
rd = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rd)
if err != nil {
return nil, err
}
req.Header.Set(pluginapi.ProtocolHeader, pluginapi.ProtocolVersion)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
if requestID != "" {
req.Header.Set(pluginapi.RequestIDHeader, requestID)
}
if c.auth != nil {
if err := c.auth.Apply(req, body); err != nil {
return nil, err
}
}
resp, err := c.http.Do(req)
if err != nil {
return nil, transport(pluginapi.CodeUnavailable, "%v", err)
}
return resp, nil
}
// decodeError turns a non-2xx answer into a protocol error.
func decodeError(resp *http.Response) error {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
var body pluginapi.ErrorBody
if json.Unmarshal(b, &body) == nil && body.Error.Code != "" {
return &body.Error
}
code := pluginapi.CodeInternal
if resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusBadGateway {
code = pluginapi.CodeUnavailable
}
return transport(code, "plugin answered HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
}
// Get reads a GET endpoint (manifest, health) into out.
func (c *Client) Get(ctx context.Context, path string, out any) error {
resp, err := c.do(ctx, http.MethodGet, path, nil, "")
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode/100 != 2 {
return decodeError(resp)
}
return json.NewDecoder(resp.Body).Decode(out)
}
// Manifest reads GET /v1/manifest.
func (c *Client) Manifest(ctx context.Context) (*pluginapi.Manifest, error) {
var m pluginapi.Manifest
return &m, c.Get(ctx, "/v1/manifest", &m)
}
// Health reads GET /v1/health.
func (c *Client) Health(ctx context.Context) error {
var h pluginapi.Health
if err := c.Get(ctx, "/v1/health", &h); err != nil {
return err
}
if h.Status != "ok" {
return &pluginapi.Error{Code: pluginapi.CodeUnavailable, Message: "plugin reports " + h.Status, Retryable: true}
}
return nil
}
func envelopeBody(env pluginapi.Envelope, input any) ([]byte, error) {
if input != nil {
raw, err := json.Marshal(input)
if err != nil {
return nil, err
}
env.Input = raw
}
return json.Marshal(env)
}
// Call posts an envelope and decodes the output into out (nil discards it).
func (c *Client) Call(ctx context.Context, path string, env pluginapi.Envelope, input, out any) error {
body, err := envelopeBody(env, input)
if err != nil {
return err
}
resp, err := c.do(ctx, http.MethodPost, path, body, env.Context.RequestID)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode/100 != 2 {
return decodeError(resp)
}
var wrapped pluginapi.Output
if err := json.NewDecoder(resp.Body).Decode(&wrapped); err != nil {
return transport(pluginapi.CodeInternal, "decode plugin output: %v", err)
}
if out == nil || len(wrapped.Output) == 0 {
return nil
}
return json.Unmarshal(wrapped.Output, out)
}
// MaxEventBytes bounds one NDJSON line (one fetched item).
const MaxEventBytes = 64 << 20
// Stream posts an envelope and hands each event to fn until the stream ends.
// It returns the "end" event's data, the "error" event as an error, or an
// error when the stream stops without either.
func (c *Client) Stream(
ctx context.Context, path string, env pluginapi.Envelope, input any,
fn func(pluginapi.Event) error,
) (json.RawMessage, error) {
body, err := envelopeBody(env, input)
if err != nil {
return nil, err
}
resp, err := c.do(ctx, http.MethodPost, path, body, env.Context.RequestID)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode/100 != 2 {
return nil, decodeError(resp)
}
sc := bufio.NewScanner(resp.Body)
sc.Buffer(make([]byte, 0, 64<<10), MaxEventBytes)
for sc.Scan() {
line := bytes.TrimSpace(sc.Bytes())
if len(line) == 0 {
continue
}
var ev pluginapi.Event
if err := json.Unmarshal(line, &ev); err != nil {
return nil, transport(pluginapi.CodeInternal, "decode stream event: %v", err)
}
switch ev.Type {
case pluginapi.EventEnd:
return ev.Data, nil
case pluginapi.EventError:
if ev.Error == nil {
return nil, &pluginapi.Error{Code: pluginapi.CodeInternal, Message: "plugin stream failed"}
}
return nil, ev.Error
}
if err := fn(ev); err != nil {
return nil, err
}
}
if err := sc.Err(); err != nil {
return nil, transport(pluginapi.CodeUnavailable, "stream broken: %v", err)
}
return nil, transport(pluginapi.CodeUnavailable, "stream ended without an end event")
}

View File

@ -0,0 +1,83 @@
// Command weknora-plugin-conformance checks a running plugin against the
// extension protocol.
//
// weknora-plugin-conformance -url https://plugin.example.com -secret $SECRET
// weknora-plugin-conformance -socket /tmp/p.sock -token $TOKEN
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"net/http"
"os"
"time"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/conformance"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
func main() {
url := flag.String("url", "", "base URL of a remote plugin")
socket := flag.String("socket", "", "unix socket of a plugin started by hand in host mode")
token := flag.String("token", "", "host token (with -socket)")
secret := flag.String("secret", "", "shared secret (with -url)")
asJSON := flag.Bool("json", false, "print the report as JSON")
timeout := flag.Duration("timeout", 30*time.Second, "per-check timeout")
flag.Parse()
var c, anon *client.Client
var raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
switch {
case *socket != "":
hs := pluginapi.Handshake{Protocol: pluginapi.ProtocolVersion, Network: "unix", Address: *socket}
c = client.ForHandshake(hs, client.Bearer(*token))
anon = client.ForHandshake(hs, nil)
case *url != "":
var auth client.Auth
if *secret != "" {
auth = client.Signed(*secret)
anon = client.New(*url, nil, nil)
}
c = client.New(*url, nil, auth)
raw = func(ctx context.Context, path string, body []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, *url+path, bytes.NewReader(body))
if err != nil {
return nil, err
}
if auth != nil {
_ = auth.Apply(req, body)
}
return http.DefaultClient.Do(req)
}
default:
fmt.Fprintln(os.Stderr, "give -url or -socket")
os.Exit(2)
}
rep := conformance.Run(context.Background(), conformance.Target{
Client: c, Unauthenticated: anon, Raw: raw, Timeout: *timeout,
})
if *asJSON {
_ = json.NewEncoder(os.Stdout).Encode(rep)
} else {
fmt.Printf("plugin %s\n", rep.Plugin)
for _, r := range rep.Results {
mark := "PASS"
if !r.Passed {
mark = "FAIL"
}
fmt.Printf(" %s %s", mark, r.Name)
if r.Detail != "" {
fmt.Printf(" — %s", r.Detail)
}
fmt.Println()
}
}
if !rep.Passed() {
os.Exit(1)
}
}

View File

@ -0,0 +1,233 @@
// Package conformance checks that a plugin speaks the extension protocol:
// it answers the discovery endpoints, rejects what it must reject, and
// answers every endpoint it claims with either a well-formed output or a
// well-formed protocol error. It does not judge the answers themselves; a
// plugin's own tests do that.
package conformance
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"regexp"
"time"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Result is one check.
type Result struct {
Name string `json:"name"`
Passed bool `json:"passed"`
Detail string `json:"detail,omitempty"`
}
// Report is every check against one plugin.
type Report struct {
Plugin string `json:"plugin"`
Results []Result `json:"results"`
}
// Passed reports whether every check passed.
func (r Report) Passed() bool {
for _, res := range r.Results {
if !res.Passed {
return false
}
}
return len(r.Results) > 0
}
// Target is the plugin under test.
type Target struct {
// Client reaches the plugin with valid credentials.
Client *client.Client
// Unauthenticated reaches it without credentials; nil skips the check
// (a plugin served without authentication in a test harness).
Unauthenticated *client.Client
// Raw posts arbitrary bytes, for malformed-request checks; nil skips.
Raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
// Timeout bounds each call (default 30s).
Timeout time.Duration
}
var semver = regexp.MustCompile(`^\d+\.\d+\.\d+([-+].*)?$`)
// Run checks a plugin.
func Run(ctx context.Context, t Target) Report {
if t.Timeout <= 0 {
t.Timeout = 30 * time.Second
}
var rep Report
check := func(name string, fn func(ctx context.Context) error) {
cctx, cancel := context.WithTimeout(ctx, t.Timeout)
defer cancel()
res := Result{Name: name, Passed: true}
if err := fn(cctx); err != nil {
res.Passed, res.Detail = false, err.Error()
}
rep.Results = append(rep.Results, res)
}
var m *pluginapi.Manifest
check("manifest", func(ctx context.Context) error {
var err error
if m, err = t.Client.Manifest(ctx); err != nil {
return err
}
rep.Plugin = m.ID + "@" + m.Version
switch {
case m.ID == "":
return fmt.Errorf("manifest has no id")
case !semver.MatchString(m.Version):
return fmt.Errorf("version %q is not semantic", m.Version)
case m.APIVersion != pluginapi.APIVersion:
return fmt.Errorf("apiVersion %q, want %q", m.APIVersion, pluginapi.APIVersion)
case len(m.Contributes) == 0:
return fmt.Errorf("manifest lists no contributions")
}
return nil
})
check("health", func(ctx context.Context) error { return t.Client.Health(ctx) })
if t.Unauthenticated != nil {
check("rejects unauthenticated calls", func(ctx context.Context) error {
return wantCode(t.Unauthenticated.Health(ctx), pluginapi.CodeUnauthorized)
})
}
check("unknown endpoint answers not_found", func(ctx context.Context) error {
return wantCode(
t.Client.Call(ctx, "/v1/no-such-endpoint", pluginapi.Envelope{}, nil, nil),
pluginapi.CodeNotFound,
)
})
check("unknown contribution answers not_found", func(ctx context.Context) error {
return wantCode(t.Client.Call(ctx, pluginapi.SearchPath("no-such-provider"), pluginapi.Envelope{},
pluginapi.SearchInput{Query: "x"}, nil), pluginapi.CodeNotFound)
})
if t.Raw != nil {
check("malformed envelope answers bad_request", func(ctx context.Context) error {
resp, err := t.Raw(ctx, "/v1/config/validate", []byte("{not json"))
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
return wantErrorBody(resp, pluginapi.CodeBadRequest)
})
}
check("config/validate answers", func(ctx context.Context) error {
return protocolAnswer(t.Client.Call(ctx, "/v1/config/validate", envelope(), nil, nil))
})
if m == nil {
return rep
}
for _, id := range m.Contributes["webSearch"] {
check("webSearch/"+id+" search answers", func(ctx context.Context) error {
var out pluginapi.SearchOutput
err := t.Client.Call(
ctx,
pluginapi.SearchPath(id),
envelope(),
pluginapi.SearchInput{Query: "weknora", MaxResults: 3},
&out,
)
if err == nil && out.Results == nil {
return fmt.Errorf("output has no results array")
}
return protocolAnswer(err)
})
}
for _, id := range m.Contributes["connectors"] {
check("connectors/"+id+" validate answers", func(ctx context.Context) error {
return protocolAnswer(t.Client.Call(ctx, pluginapi.ConnectorValidatePath(id), envelope(), nil, nil))
})
check("connectors/"+id+" list-resources answers", func(ctx context.Context) error {
var out pluginapi.ListResourcesOutput
err := t.Client.Call(
ctx,
pluginapi.ConnectorListResourcesPath(id),
envelope(),
pluginapi.ListResourcesInput{},
&out,
)
if err == nil && out.Resources == nil {
return fmt.Errorf("output has no resources array")
}
return protocolAnswer(err)
})
check("connectors/"+id+" resolve-ancestors answers", func(ctx context.Context) error {
var out pluginapi.ResolveAncestorsOutput
err := t.Client.Call(ctx, pluginapi.ConnectorResolveAncestorsPath(id), envelope(),
pluginapi.ResolveAncestorsInput{ResourceIDs: []string{}}, &out)
if err == nil && out.Ancestors == nil {
return fmt.Errorf("output has no ancestors array")
}
return protocolAnswer(err)
})
check("connectors/"+id+" fetch stream terminates", func(ctx context.Context) error {
_, err := t.Client.Stream(ctx, pluginapi.ConnectorFetchPath(id), envelope(),
pluginapi.FetchInput{Mode: pluginapi.FetchFull}, func(ev pluginapi.Event) error {
switch ev.Type {
case pluginapi.EventItem, pluginapi.EventCheckpoint, pluginapi.EventProgress, pluginapi.EventLog:
return nil
}
return fmt.Errorf("unknown event type %q", ev.Type)
})
return protocolAnswer(err)
})
}
return rep
}
// envelope is a call with empty configuration: plugins must answer it with
// an output or a protocol error, never a crash or garbage.
func envelope() pluginapi.Envelope {
return pluginapi.Envelope{Context: pluginapi.Context{TenantID: 1, RequestID: "conformance", Locale: "en-US"}}
}
// protocolAnswer accepts success or a protocol error the plugin chose, and
// fails transport failures and internal errors: a plugin must answer an
// empty configuration with invalid_config, not by crashing.
func protocolAnswer(err error) error {
if err == nil {
return nil
}
var te *client.TransportError
if errors.As(err, &te) {
return fmt.Errorf("no protocol answer: %v", te)
}
e, ok := pluginapi.AsError(err)
if !ok {
return err
}
if e.Code == pluginapi.CodeInternal {
return fmt.Errorf("answered internal: %s", e.Message)
}
return nil
}
func wantCode(err error, code pluginapi.ErrorCode) error {
e, ok := pluginapi.AsError(err)
if !ok {
return fmt.Errorf("want a %s protocol error, got %v", code, err)
}
if e.Code != code {
return fmt.Errorf("want %s, got %s: %s", code, e.Code, e.Message)
}
return nil
}
func wantErrorBody(resp *http.Response, code pluginapi.ErrorCode) error {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
var body pluginapi.ErrorBody
if err := json.Unmarshal(b, &body); err != nil {
return fmt.Errorf("HTTP %d without a protocol error body: %s", resp.StatusCode, b)
}
if resp.StatusCode != code.HTTPStatus() || body.Error.Code != code {
return fmt.Errorf("want HTTP %d %s, got HTTP %d %s", code.HTTPStatus(), code, resp.StatusCode, body.Error.Code)
}
return nil
}

View File

@ -0,0 +1,105 @@
package conformance_test
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/pluginsdk"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/conformance"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
type strictConnector struct{}
func (strictConnector) Validate(_ context.Context, _ *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
if cfg.Settings["url"] == nil {
return pluginapi.InvalidConfig("url is required", map[string]string{"settings.url": "required"})
}
return nil
}
func (s strictConnector) ListResources(
ctx context.Context,
call *pluginsdk.Call,
cfg pluginsdk.ConnectorConfig,
_ string,
) ([]pluginapi.Resource, error) {
return nil, s.Validate(ctx, call, cfg)
}
func (s strictConnector) Fetch(
ctx context.Context,
call *pluginsdk.Call,
cfg pluginsdk.ConnectorConfig,
_ pluginapi.FetchInput,
_ *pluginsdk.Stream,
) (*pluginapi.Cursor, error) {
return nil, s.Validate(ctx, call, cfg)
}
func target(t *testing.T, h http.Handler) conformance.Target {
srv := httptest.NewServer(h)
t.Cleanup(srv.Close)
return conformance.Target{
Client: client.New(srv.URL, nil, nil),
Raw: func(ctx context.Context, path string, body []byte) (*http.Response, error) {
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+path, bytes.NewReader(body))
return http.DefaultClient.Do(req)
},
}
}
func TestSDKPluginConforms(t *testing.T) {
p := pluginsdk.New(pluginsdk.Info{ID: "acme.strict", Version: "1.0.0"})
p.Connector("strict", strictConnector{})
p.WebSearch(
"none",
pluginsdk.WebSearchFunc(
func(context.Context, *pluginsdk.Call, pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
return nil, pluginapi.InvalidConfig("api key required", map[string]string{"api_key": "required"})
},
),
)
rep := conformance.Run(context.Background(), target(t, p.Handler()))
for _, r := range rep.Results {
if !r.Passed {
t.Errorf("%s: %s", r.Name, r.Detail)
}
}
if !rep.Passed() || rep.Plugin != "acme.strict@1.0.0" || len(rep.Results) < 10 {
t.Fatalf("report = %+v", rep)
}
}
func TestBrokenPluginFails(t *testing.T) {
p := pluginsdk.New(pluginsdk.Info{ID: "acme.broken", Version: "1.0.0"})
p.Connector("broken", strictConnector{})
good := p.Handler()
// A fetch that returns plain text and a list that crashes.
broken := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/fetch"):
_, _ = w.Write([]byte("oops"))
case strings.HasSuffix(r.URL.Path, "/list-resources"):
panic("nil map")
default:
good.ServeHTTP(w, r)
}
})
rep := conformance.Run(context.Background(), target(t, broken))
failed := map[string]bool{}
for _, r := range rep.Results {
if !r.Passed {
failed[r.Name] = true
}
}
if rep.Passed() || !failed["connectors/broken fetch stream terminates"] ||
!failed["connectors/broken list-resources answers"] {
t.Fatalf("failed checks = %v", failed)
}
}

View File

@ -0,0 +1,160 @@
package pluginsdk
import (
"context"
"encoding/json"
"net/http"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// ConnectorConfig is a data source's instance configuration.
type ConnectorConfig struct {
Credentials map[string]any `json:"credentials"`
Settings map[string]any `json:"settings"`
ResourceIDs []string `json:"resourceIds"`
}
// Connector is a data source connector.
type Connector interface {
// Validate checks the configuration, typically by calling the source.
// Return pluginapi.InvalidConfig to point at fields.
Validate(ctx context.Context, call *Call, cfg ConnectorConfig) error
// ListResources lists what can be synced under parentID ("" = top).
ListResources(ctx context.Context, call *Call, cfg ConnectorConfig, parentID string) ([]pluginapi.Resource, error)
// Fetch streams items changed since cursor (nil = everything) and
// returns the cursor for the next sync. Checkpoint at page boundaries
// so an interrupted sync resumes.
Fetch(
ctx context.Context,
call *Call,
cfg ConnectorConfig,
in pluginapi.FetchInput,
s *Stream,
) (*pluginapi.Cursor, error)
}
// AncestorResolver is implemented by connectors whose resource tree loads
// lazily; see pluginapi.ResolveAncestorsInput. Others answer with nothing.
type AncestorResolver interface {
ResolveAncestors(ctx context.Context, call *Call, cfg ConnectorConfig, resourceIDs []string) ([]string, error)
}
// Connector registers connector id (contributes.connectors[].id).
func (p *Plugin) Connector(id string, c Connector) { p.connectors[id] = c }
func (p *Plugin) connector(w http.ResponseWriter, r *http.Request) (Connector, bool) {
c, ok := p.connectors[r.PathValue("id")]
if !ok {
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no connector %q", r.PathValue("id")))
}
return c, ok
}
func connectorConfig(call *Call) (ConnectorConfig, error) {
var cfg ConnectorConfig
if err := call.DecodeInstance(&cfg); err != nil {
return cfg, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode connector config: %v", err)
}
return cfg, nil
}
func (p *Plugin) routeConnectors(mux *http.ServeMux) {
mux.HandleFunc("POST /v1/connectors/{id}/validate", func(w http.ResponseWriter, r *http.Request) {
c, ok := p.connector(w, r)
if !ok {
return
}
p.unary(func(ctx context.Context, call *Call, _ json.RawMessage) (any, error) {
cfg, err := connectorConfig(call)
if err != nil {
return nil, err
}
return struct{}{}, c.Validate(ctx, call, cfg)
})(w, r)
})
mux.HandleFunc("POST /v1/connectors/{id}/list-resources", func(w http.ResponseWriter, r *http.Request) {
c, ok := p.connector(w, r)
if !ok {
return
}
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
in, err := decodeInput[pluginapi.ListResourcesInput](raw)
if err != nil {
return nil, err
}
cfg, err := connectorConfig(call)
if err != nil {
return nil, err
}
res, err := c.ListResources(ctx, call, cfg, in.ParentID)
if err != nil {
return nil, err
}
if res == nil {
res = []pluginapi.Resource{}
}
return pluginapi.ListResourcesOutput{Resources: res}, nil
})(w, r)
})
mux.HandleFunc("POST /v1/connectors/{id}/resolve-ancestors", func(w http.ResponseWriter, r *http.Request) {
c, ok := p.connector(w, r)
if !ok {
return
}
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
in, err := decodeInput[pluginapi.ResolveAncestorsInput](raw)
if err != nil {
return nil, err
}
out := pluginapi.ResolveAncestorsOutput{Ancestors: []string{}}
resolver, ok := c.(AncestorResolver)
if !ok {
return out, nil
}
cfg, err := connectorConfig(call)
if err != nil {
return nil, err
}
ids, err := resolver.ResolveAncestors(ctx, call, cfg, in.ResourceIDs)
if err != nil {
return nil, err
}
if ids != nil {
out.Ancestors = ids
}
return out, nil
})(w, r)
})
mux.HandleFunc("POST /v1/connectors/{id}/fetch", func(w http.ResponseWriter, r *http.Request) {
c, ok := p.connector(w, r)
if !ok {
return
}
env, err := readEnvelope(r)
if err != nil {
writeError(w, err)
return
}
in, err := decodeInput[pluginapi.FetchInput](env.Input)
if err != nil {
writeError(w, err)
return
}
call := &Call{Context: env.Context, Config: env.Config}
cfg, err := connectorConfig(call)
if err != nil {
writeError(w, err)
return
}
ctx, cancel := callContext(r.Context(), env)
defer cancel()
s := newStream(w)
cursor, err := c.Fetch(ctx, call, cfg, in, s)
if err != nil {
s.fail(err)
return
}
s.end(cursor)
})
}

3
pluginsdk/go.mod 100644
View File

@ -0,0 +1,3 @@
module github.com/Tencent/WeKnora/pluginsdk
go 1.24

View File

@ -0,0 +1,49 @@
package pluginsdk
import (
"os"
"regexp"
"sort"
"strings"
"testing"
)
// The OpenAPI description and the SDK's routes must list the same endpoints,
// so a protocol change cannot land in one and not the other.
func TestOpenAPIMatchesRoutes(t *testing.T) {
raw, err := os.ReadFile("pluginapi/openapi.yaml")
if err != nil {
t.Fatal(err)
}
// Windows checkouts may carry CRLF line endings.
spec := strings.ReplaceAll(string(raw), "\r\n", "\n")
var documented []string
for _, m := range regexp.MustCompile(`(?m)^ (/v1/[^:]+):$`).FindAllStringSubmatch(spec, -1) {
documented = append(documented, m[1])
}
src, err := os.ReadFile("plugin.go")
if err != nil {
t.Fatal(err)
}
for _, f := range []string{"websearch.go", "connector.go"} {
b, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
src = append(src, b...)
}
var routed []string
route := regexp.MustCompile(`HandleFunc\(\s*"(?:GET|POST) (/v1/[^"]+)"`)
for _, m := range route.FindAllStringSubmatch(string(src), -1) {
routed = append(routed, m[1])
}
sort.Strings(documented)
sort.Strings(routed)
if strings.Join(documented, "\n") != strings.Join(routed, "\n") {
t.Fatalf(
"openapi.yaml documents:\n%s\n\nthe SDK routes:\n%s",
strings.Join(documented, "\n"),
strings.Join(routed, "\n"),
)
}
}

349
pluginsdk/plugin.go 100644
View File

@ -0,0 +1,349 @@
// Package pluginsdk builds WeKnora code plugins in Go. A plugin registers what
// it contributes and calls Serve; the SDK speaks the extension protocol
// (package pluginapi): the handshake with the plugin host, authentication,
// routing, streaming and error encoding.
//
// func main() {
// p := pluginsdk.New(pluginsdk.Info{ID: "acme.rss", Version: "1.0.0"})
// p.Connector("rss", &rssConnector{})
// if err := p.Serve(); err != nil {
// log.Fatal(err)
// }
// }
//
// Started by the WeKnora plugin host, the plugin listens where the host says
// and accepts only the host's token. Run on its own (a remote plugin), it
// listens on WEKNORA_PLUGIN_ADDR and verifies request signatures with
// WEKNORA_PLUGIN_SECRET.
package pluginsdk
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"sort"
"strings"
"syscall"
"time"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Info identifies the plugin; it must match plugin.yaml.
type Info struct {
ID string
Version string
}
// Call is one request's context and configuration. Configuration secrets are
// already decrypted; do not keep them beyond the call.
type Call struct {
pluginapi.Context
Config pluginapi.Config
}
// DecodeInstance decodes the instance configuration into v.
func (c *Call) DecodeInstance(v any) error {
return remarshal(c.Config.Instance, v)
}
// ConfigValidator checks the plugin's system and tenant configuration.
type ConfigValidator func(ctx context.Context, call *Call) error
// Plugin is a plugin under construction; register contributions, then Serve.
type Plugin struct {
info Info
webSearch map[string]WebSearcher
connectors map[string]Connector
validate ConfigValidator
logger *slog.Logger
// ShutdownTimeout bounds how long Serve waits for calls in flight after
// a termination signal.
ShutdownTimeout time.Duration
}
// New starts a plugin.
func New(info Info) *Plugin {
return &Plugin{
info: info,
webSearch: map[string]WebSearcher{},
connectors: map[string]Connector{},
logger: slog.New(slog.NewTextHandler(os.Stderr, nil)),
ShutdownTimeout: 60 * time.Second,
}
}
// Logger writes to stderr, which the plugin host forwards to WeKnora's logs.
func (p *Plugin) Logger() *slog.Logger { return p.logger }
// ValidateConfig registers the check behind POST /v1/config/validate.
func (p *Plugin) ValidateConfig(fn ConfigValidator) { p.validate = fn }
// Manifest is what the plugin reports at GET /v1/manifest.
func (p *Plugin) Manifest() pluginapi.Manifest {
m := pluginapi.Manifest{
ID: p.info.ID, Version: p.info.Version, APIVersion: pluginapi.APIVersion,
Contributes: map[string][]string{},
}
add := func(point string, ids []string) {
if len(ids) > 0 {
sort.Strings(ids)
m.Contributes[point] = ids
}
}
add("webSearch", keys(p.webSearch))
add("connectors", keys(p.connectors))
return m
}
func keys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
// Handler serves the protocol. Serve wraps it with authentication; tests
// and custom servers can use it directly.
func (p *Plugin) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /v1/manifest", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, p.Manifest())
})
mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, pluginapi.Health{Status: "ok"})
})
mux.HandleFunc(
"POST /v1/config/validate",
p.unary(func(ctx context.Context, call *Call, _ json.RawMessage) (any, error) {
if p.validate != nil {
if err := p.validate(ctx, call); err != nil {
return nil, err
}
}
return struct{}{}, nil
}),
)
p.routeWebSearch(mux)
p.routeConnectors(mux)
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no endpoint %s %s", r.Method, r.URL.Path))
})
return recoverer(p.logger, mux)
}
// unaryFunc answers one call with one output.
type unaryFunc func(ctx context.Context, call *Call, input json.RawMessage) (any, error)
func (p *Plugin) unary(fn unaryFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
env, err := readEnvelope(r)
if err != nil {
writeError(w, err)
return
}
ctx, cancel := callContext(r.Context(), env)
defer cancel()
out, err := fn(ctx, &Call{Context: env.Context, Config: env.Config}, env.Input)
if err != nil {
writeError(w, err)
return
}
raw, err := json.Marshal(out)
if err != nil {
writeError(w, err)
return
}
writeJSON(w, http.StatusOK, pluginapi.Output{Output: raw})
}
}
func readEnvelope(r *http.Request) (*pluginapi.Envelope, error) {
var env pluginapi.Envelope
if err := json.NewDecoder(r.Body).Decode(&env); err != nil {
return nil, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode envelope: %v", err)
}
return &env, nil
}
// callContext applies the envelope deadline.
func callContext(parent context.Context, env *pluginapi.Envelope) (context.Context, context.CancelFunc) {
if env.Context.Deadline != nil {
return context.WithDeadline(parent, *env.Context.Deadline)
}
return context.WithCancel(parent)
}
func decodeInput[T any](raw json.RawMessage) (T, error) {
var v T
if len(raw) == 0 {
return v, nil
}
if err := json.Unmarshal(raw, &v); err != nil {
return v, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode input: %v", err)
}
return v, nil
}
func remarshal(from, to any) error {
b, err := json.Marshal(from)
if err != nil {
return err
}
return json.Unmarshal(b, to)
}
func writeJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(v)
}
// toError maps any error to a protocol error.
func toError(err error) *pluginapi.Error {
if e, ok := pluginapi.AsError(err); ok {
return e
}
if errors.Is(err, context.DeadlineExceeded) {
return pluginapi.Errorf(pluginapi.CodeUnavailable, "deadline exceeded")
}
return &pluginapi.Error{Code: pluginapi.CodeInternal, Message: err.Error()}
}
func writeError(w http.ResponseWriter, err error) {
e := toError(err)
writeJSON(w, e.Code.HTTPStatus(), pluginapi.ErrorBody{Error: *e})
}
func recoverer(logger *slog.Logger, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if v := recover(); v != nil {
logger.Error("plugin panicked", "path", r.URL.Path, "panic", v)
writeError(w, fmt.Errorf("plugin panicked: %v", v))
}
}()
next.ServeHTTP(w, r)
})
}
// Serve runs the plugin until SIGTERM or SIGINT, then drains calls in
// flight. Started by the plugin host it listens where the host says and
// prints the handshake; otherwise it serves as a remote plugin.
func (p *Plugin) Serve() error {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
defer stop()
return p.ServeContext(ctx)
}
// ServeContext is Serve with a caller-controlled lifetime.
func (p *Plugin) ServeContext(ctx context.Context) error {
ln, auth, handshake, err := p.listen()
if err != nil {
return err
}
srv := &http.Server{Handler: auth(p.Handler()), ReadHeaderTimeout: 10 * time.Second}
errc := make(chan error, 1)
go func() { errc <- srv.Serve(ln) }()
if handshake != nil {
// The host waits for this exact line; everything else on stdout is
// treated as log output.
_, _ = fmt.Fprintln(os.Stdout, handshake.String())
} else {
p.logger.Info("plugin serving", "addr", ln.Addr().String())
}
select {
case err := <-errc:
return err
case <-ctx.Done():
}
shutdown, cancel := context.WithTimeout(context.Background(), p.ShutdownTimeout)
defer cancel()
return srv.Shutdown(shutdown)
}
type middleware func(http.Handler) http.Handler
// listen picks host mode (socket + token from the environment) or remote
// mode (address + signing secret).
func (p *Plugin) listen() (net.Listener, middleware, *pluginapi.Handshake, error) {
if addr := os.Getenv(pluginapi.EnvSocket); addr != "" {
network := os.Getenv(pluginapi.EnvNetwork)
if network == "" {
network = "unix"
}
token := os.Getenv(pluginapi.EnvToken)
if token == "" {
return nil, nil, nil, fmt.Errorf("%s is set without %s", pluginapi.EnvSocket, pluginapi.EnvToken)
}
if network == "unix" {
_ = os.Remove(addr)
}
ln, err := net.Listen(network, addr)
if err != nil {
return nil, nil, nil, fmt.Errorf("listen %s %s: %w", network, addr, err)
}
hs := &pluginapi.Handshake{Protocol: pluginapi.ProtocolVersion, Network: network, Address: ln.Addr().String()}
return ln, bearerAuth(token), hs, nil
}
addr := os.Getenv(pluginapi.EnvAddr)
if addr == "" {
addr = ":8080"
}
secret := os.Getenv(pluginapi.EnvSecret)
if secret == "" {
return nil, nil, nil, fmt.Errorf(
"a remote plugin needs %s (the secret shown when it was registered)",
pluginapi.EnvSecret,
)
}
ln, err := net.Listen("tcp", addr)
if err != nil {
return nil, nil, nil, err
}
return ln, signatureAuth([]byte(secret)), nil, nil
}
func bearerAuth(token string) middleware {
want := "Bearer " + token
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != want {
writeError(w, pluginapi.Errorf(pluginapi.CodeUnauthorized, "missing or wrong host token"))
return
}
next.ServeHTTP(w, r)
})
}
}
// maxBody bounds a signed request body read into memory for verification.
const maxBody = 32 << 20
func signatureAuth(secret []byte) middleware {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(io.LimitReader(r.Body, maxBody+1))
if err != nil || len(body) > maxBody {
writeError(w, pluginapi.Errorf(pluginapi.CodeBadRequest, "request body unreadable or too large"))
return
}
err = pluginapi.VerifySignature(secret, r.Header.Get(pluginapi.TimestampHeader),
r.Header.Get(pluginapi.SignatureHeader), body, time.Now())
if err != nil {
writeError(w, pluginapi.Errorf(pluginapi.CodeUnauthorized, "%v", err))
return
}
r.Body = io.NopCloser(strings.NewReader(string(body)))
next.ServeHTTP(w, r)
})
}
}

View File

@ -0,0 +1,259 @@
package pluginsdk
import (
"bufio"
"context"
"errors"
"fmt"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/pluginsdk/client"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
type feedConnector struct{ items int }
func (f *feedConnector) Validate(_ context.Context, _ *Call, cfg ConnectorConfig) error {
if cfg.Settings["url"] == nil {
return pluginapi.InvalidConfig("feed URL is required", map[string]string{"settings.url": "required"})
}
return nil
}
func (f *feedConnector) ListResources(
_ context.Context,
_ *Call,
cfg ConnectorConfig,
parentID string,
) ([]pluginapi.Resource, error) {
if parentID != "" {
return nil, nil
}
return []pluginapi.Resource{{ExternalID: fmt.Sprint(cfg.Settings["url"]), Name: "Feed"}}, nil
}
func (f *feedConnector) Fetch(
_ context.Context, _ *Call, _ ConnectorConfig, in pluginapi.FetchInput, s *Stream,
) (*pluginapi.Cursor, error) {
start := 0
if in.Cursor != nil {
start = int(in.Cursor.State["next"].(float64))
}
for i := start; i < f.items; i++ {
item := pluginapi.FetchedItem{ExternalID: fmt.Sprint(i), Title: fmt.Sprint("item ", i), Content: []byte("body")}
if err := s.Item(item); err != nil {
return nil, err
}
if err := s.Checkpoint(pluginapi.Cursor{State: map[string]any{"next": i + 1}}); err != nil {
return nil, err
}
if i == 2 && in.Mode == "fail-midway" {
return nil, pluginapi.Errorf(pluginapi.CodeUnavailable, "upstream went away")
}
}
return &pluginapi.Cursor{State: map[string]any{"next": f.items}}, nil
}
func testPlugin(t *testing.T) (*Plugin, *client.Client) {
t.Helper()
p := New(Info{ID: "acme.feed", Version: "1.0.0"})
p.Connector("feed", &feedConnector{items: 5})
p.WebSearch(
"echo",
WebSearchFunc(func(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
switch in.Query {
case "panic":
panic("boom")
case "deadline":
if ctx.Err() != nil {
return nil, ctx.Err()
}
}
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{
Title: in.Query,
URL: fmt.Sprint("https://example.com/?tenant=", call.TenantID,
"&key=", call.Config.Instance["api_key"]),
}}}, nil
}),
)
srv := httptest.NewServer(p.Handler())
t.Cleanup(srv.Close)
return p, client.New(srv.URL, nil, nil)
}
func TestUnaryCallsAndErrors(t *testing.T) {
ctx := context.Background()
_, c := testPlugin(t)
m, err := c.Manifest(ctx)
if err != nil || m.ID != "acme.feed" || m.Contributes["connectors"][0] != "feed" ||
m.APIVersion != pluginapi.APIVersion {
t.Fatalf("manifest = %+v, %v", m, err)
}
if err := c.Health(ctx); err != nil {
t.Fatal(err)
}
env := pluginapi.Envelope{
Context: pluginapi.Context{TenantID: 7},
Config: pluginapi.Config{Instance: map[string]any{"api_key": "k1"}},
}
var out pluginapi.SearchOutput
if err := c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "hi"}, &out); err != nil {
t.Fatal(err)
}
if len(out.Results) != 1 || out.Results[0].URL != "https://example.com/?tenant=7&key=k1" {
t.Fatalf("search = %+v", out)
}
err = c.Call(ctx, pluginapi.ConnectorValidatePath("feed"), pluginapi.Envelope{}, nil, nil)
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeInvalidConfig ||
e.Details.Fields["settings.url"] != "required" {
t.Fatalf("validate = %v", err)
}
err = c.Call(ctx, pluginapi.SearchPath("nope"), env, pluginapi.SearchInput{}, nil)
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeNotFound {
t.Fatalf("unknown provider = %v", err)
}
err = c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "panic"}, nil)
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeInternal ||
!strings.Contains(e.Message, "boom") {
t.Fatalf("panic = %v", err)
}
past := time.Now().Add(-time.Second)
env.Context.Deadline = &past
err = c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "deadline"}, nil)
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeUnavailable {
t.Fatalf("the envelope deadline must reach the handler's context, got %v", err)
}
}
func TestStreamingFetch(t *testing.T) {
ctx := context.Background()
_, c := testPlugin(t)
env := pluginapi.Envelope{
Config: pluginapi.Config{Instance: map[string]any{"settings": map[string]any{"url": "u"}}},
}
var items []string
var last *pluginapi.Cursor
end, err := c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env,
pluginapi.FetchInput{Mode: pluginapi.FetchFull, Cursor: &pluginapi.Cursor{State: map[string]any{"next": 3}}},
func(ev pluginapi.Event) error {
switch ev.Type {
case pluginapi.EventItem:
var it pluginapi.FetchedItem
if err := remarshal(ev.Data, &it); err != nil {
return err
}
items = append(items, it.ExternalID+":"+string(it.Content))
case pluginapi.EventCheckpoint:
last = &pluginapi.Cursor{}
return remarshal(ev.Data, last)
}
return nil
})
if err != nil {
t.Fatal(err)
}
if strings.Join(items, ",") != "3:body,4:body" || last.State["next"].(float64) != 5 ||
!strings.Contains(string(end), `"next":5`) {
t.Fatalf("items=%v last=%+v end=%s", items, last, end)
}
_, err = c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env, pluginapi.FetchInput{Mode: "fail-midway"},
func(pluginapi.Event) error { return nil })
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeUnavailable || !e.Retryable {
t.Fatalf("mid-stream failure = %v", err)
}
stop := errors.New("ingest failed")
_, err = c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env, pluginapi.FetchInput{},
func(pluginapi.Event) error { return stop })
if !errors.Is(err, stop) {
t.Fatalf("a handler error must abort the stream, got %v", err)
}
}
func TestAuthModes(t *testing.T) {
ctx := context.Background()
p, _ := testPlugin(t)
bearer := httptest.NewServer(bearerAuth("tok")(p.Handler()))
defer bearer.Close()
if err := client.New(bearer.URL, nil, client.Bearer("wrong")).Health(ctx); !isCode(
err,
pluginapi.CodeUnauthorized,
) {
t.Fatalf("wrong token = %v", err)
}
if err := client.New(bearer.URL, nil, client.Bearer("tok")).Health(ctx); err != nil {
t.Fatal(err)
}
signed := httptest.NewServer(signatureAuth([]byte("s3cret"))(p.Handler()))
defer signed.Close()
if err := client.New(signed.URL, nil, client.Signed("other")).Health(ctx); !isCode(
err,
pluginapi.CodeUnauthorized,
) {
t.Fatalf("wrong secret = %v", err)
}
var out pluginapi.SearchOutput
err := client.New(signed.URL, nil, client.Signed("s3cret")).
Call(ctx, pluginapi.SearchPath("echo"), pluginapi.Envelope{}, pluginapi.SearchInput{Query: "q"}, &out)
if err != nil || len(out.Results) != 1 {
t.Fatalf("signed call = %+v, %v", out, err)
}
}
func isCode(err error, code pluginapi.ErrorCode) bool {
e, ok := pluginapi.AsError(err)
return ok && e.Code == code
}
// ServeContext in host mode listens on the socket from the environment,
// prints the handshake and requires the token.
func TestServeHostHandshake(t *testing.T) {
dir, err := os.MkdirTemp("", "wkp")
if err != nil {
t.Fatal(err)
}
defer func() { _ = os.RemoveAll(dir) }()
sock := filepath.Join(dir, "p.sock")
t.Setenv(pluginapi.EnvSocket, sock)
t.Setenv(pluginapi.EnvToken, "tok")
r, w, _ := os.Pipe()
stdout := os.Stdout
os.Stdout = w
defer func() { os.Stdout = stdout }()
p, _ := testPlugin(t)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- p.ServeContext(ctx) }()
line, err := bufio.NewReader(r).ReadString('\n')
if err != nil {
t.Fatal(err)
}
os.Stdout = stdout
hs, ok, err := pluginapi.ParseHandshake(line)
if !ok || err != nil || hs.Network != "unix" || hs.Address != sock {
t.Fatalf("handshake %q = %+v %v %v", line, hs, ok, err)
}
c := client.ForHandshake(hs, client.Bearer("tok"))
if err := c.Health(context.Background()); err != nil {
t.Fatal(err)
}
cancel()
if err := <-done; err != nil {
t.Fatalf("shutdown: %v", err)
}
}

View File

@ -0,0 +1,96 @@
package pluginapi
import "time"
// ConnectorValidatePath is the validate endpoint of connector id.
func ConnectorValidatePath(id string) string { return "/v1/connectors/" + id + "/validate" }
// ConnectorListResourcesPath is the list-resources endpoint of connector id.
func ConnectorListResourcesPath(id string) string { return "/v1/connectors/" + id + "/list-resources" }
// ConnectorResolveAncestorsPath is the resolve-ancestors endpoint of connector id.
func ConnectorResolveAncestorsPath(id string) string {
return "/v1/connectors/" + id + "/resolve-ancestors"
}
// ConnectorFetchPath is the streaming fetch endpoint of connector id.
func ConnectorFetchPath(id string) string { return "/v1/connectors/" + id + "/fetch" }
// The connector's instance configuration arrives in Envelope.Config.Instance
// as {"credentials": {...}, "settings": {...}, "resourceIds": [...]}.
// ListResourcesInput asks for the resources under ParentID ("" = top level).
type ListResourcesInput struct {
ParentID string `json:"parentId,omitempty"`
}
// ListResourcesOutput lists resources a user can pick for syncing.
type ListResourcesOutput struct {
Resources []Resource `json:"resources"`
}
// Resource is something a data source can sync (a space, folder, feed).
type Resource struct {
ExternalID string `json:"externalId"`
Name string `json:"name"`
Type string `json:"type,omitempty"`
Description string `json:"description,omitempty"`
URL string `json:"url,omitempty"`
ModifiedAt *time.Time `json:"modifiedAt,omitempty"`
ParentID string `json:"parentId,omitempty"`
HasChildren bool `json:"hasChildren,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
// ResolveAncestorsInput asks which ancestors of the given resources a
// lazily loaded picker must expand.
type ResolveAncestorsInput struct {
ResourceIDs []string `json:"resourceIds"`
}
// ResolveAncestorsOutput is the deduplicated set of ancestor IDs.
type ResolveAncestorsOutput struct {
Ancestors []string `json:"ancestors"`
}
// Fetch modes.
const (
FetchFull = "full"
FetchIncremental = "incremental"
)
// FetchInput starts or resumes a sync. The answer is a stream of "item"
// events (FetchedItem), "checkpoint" events (Cursor) at page boundaries and
// an "end" event carrying the final Cursor.
type FetchInput struct {
Mode string `json:"mode"`
// Cursor is the last checkpoint or final cursor; nil starts from scratch.
Cursor *Cursor `json:"cursor,omitempty"`
// ResourceIDs are the selected resources (also in config.instance).
ResourceIDs []string `json:"resourceIds,omitempty"`
}
// Cursor is the connector's resumable state. WeKnora stores it and hands it
// back; its content is the plugin's business. A checkpoint must be a
// complete snapshot: resuming from it reproduces all progress so far.
type Cursor struct {
LastSyncTime *time.Time `json:"lastSyncTime,omitempty"`
State map[string]any `json:"state,omitempty"`
}
// FetchedItem is one document fetched from the source.
type FetchedItem struct {
ExternalID string `json:"externalId"`
Title string `json:"title"`
// Content is the document body; Markdown preferred. JSON carries it as
// base64.
Content []byte `json:"content,omitempty"`
ContentType string `json:"contentType,omitempty"`
FileName string `json:"fileName,omitempty"`
URL string `json:"url,omitempty"`
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
CreatedAt *time.Time `json:"createdAt,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
IsDeleted bool `json:"isDeleted,omitempty"`
SourceResourceID string `json:"sourceResourceId,omitempty"`
}

View File

@ -0,0 +1,32 @@
// Package pluginapi is version 1 of the WeKnora extension protocol: the
// messages WeKnora and a code plugin exchange over HTTP. It has no
// dependencies so that plugins and WeKnora share one definition; openapi.yaml
// in this directory describes the same protocol for other languages.
//
// Every call is a POST whose body is an Envelope and whose successful answer
// is {"output": ...}. Streaming calls answer with NDJSON Events instead. A
// failed call answers with a non-2xx status and an ErrorBody. Within a major
// version the protocol only grows: new optional fields and new endpoints.
// Both sides ignore fields they do not know.
package pluginapi
// ProtocolVersion is the major version this package speaks; it travels in
// the ProtocolHeader of every request.
const ProtocolVersion = "1"
// APIVersion is the manifest apiVersion of plugins speaking this protocol.
const APIVersion = "weknora.plugin/v1"
// Request headers.
const (
ProtocolHeader = "X-WeKnora-Protocol"
RequestIDHeader = "X-Request-Id"
// SignatureHeader and TimestampHeader authenticate calls to remote
// plugins: hex HMAC-SHA256 over "<timestamp>.<body>" with the shared
// secret. Host plugins get a bearer token instead.
SignatureHeader = "X-WeKnora-Signature"
TimestampHeader = "X-WeKnora-Timestamp"
)
// NDJSONContentType is the content type of streaming answers.
const NDJSONContentType = "application/x-ndjson"

View File

@ -0,0 +1,71 @@
package pluginapi
import (
"encoding/json"
"time"
)
// Envelope is the body of every call. Plugins must be stateless: everything
// about the tenant comes in the envelope, and credentials must not be cached
// across calls.
type Envelope struct {
Context Context `json:"context"`
Config Config `json:"config"`
// Input is the endpoint-specific request, such as a SearchInput.
Input json.RawMessage `json:"input,omitempty"`
}
// Context describes who is calling and until when.
type Context struct {
TenantID uint64 `json:"tenantId"`
UserID string `json:"userId,omitempty"`
Locale string `json:"locale,omitempty"`
RequestID string `json:"requestId,omitempty"`
// Deadline is when WeKnora stops waiting; plugins should give up by then.
Deadline *time.Time `json:"deadline,omitempty"`
// Host lets the plugin call back into WeKnora (Host API); absent when
// the plugin was granted no scopes.
Host *HostAccess `json:"host,omitempty"`
}
// HostAccess is where and how a plugin calls the WeKnora Host API.
type HostAccess struct {
URL string `json:"url"`
// Token is a short-lived bearer token bound to this tenant and the
// plugin's granted scopes.
Token string `json:"token"`
}
// Config carries the three configuration scopes separately, secrets already
// decrypted. They are never merged, so a key means the same thing wherever
// it appears.
type Config struct {
// System is the platform-wide configuration (config.system).
System map[string]any `json:"system,omitempty"`
// Tenant is the workspace configuration (config.tenant).
Tenant map[string]any `json:"tenant,omitempty"`
// Instance is the configuration of the integration instance being used:
// a data source, a web search provider.
Instance map[string]any `json:"instance,omitempty"`
}
// Output wraps a successful answer.
type Output struct {
Output json.RawMessage `json:"output"`
}
// Manifest is what GET /v1/manifest answers: enough for WeKnora to check that
// the process it started is the package it installed.
type Manifest struct {
ID string `json:"id"`
Version string `json:"version"`
APIVersion string `json:"apiVersion"`
// Contributes lists the contribution IDs the process serves per
// extension point, e.g. {"connectors": ["rss"]}.
Contributes map[string][]string `json:"contributes"`
}
// Health is what GET /v1/health answers.
type Health struct {
Status string `json:"status"` // "ok"
}

View File

@ -0,0 +1,90 @@
package pluginapi
import (
"errors"
"fmt"
)
// ErrorCode classifies a failed call; WeKnora reacts to each differently.
type ErrorCode string
const (
// CodeInvalidConfig means the configuration is wrong; Details.Fields says
// where. WeKnora shows it on the form.
CodeInvalidConfig ErrorCode = "invalid_config"
// CodeUnauthorized means the credentials no longer work.
CodeUnauthorized ErrorCode = "unauthorized"
// CodeNotFound means the resource does not exist.
CodeNotFound ErrorCode = "not_found"
// CodeRateLimited means back off for Details.RetryAfter seconds.
CodeRateLimited ErrorCode = "rate_limited"
// CodeUnavailable means the upstream service is down; retry later.
CodeUnavailable ErrorCode = "unavailable"
// CodeBadRequest means the call itself is malformed (protocol error).
CodeBadRequest ErrorCode = "bad_request"
// CodeInternal means the plugin failed.
CodeInternal ErrorCode = "internal"
)
// Error is a failed call. Plugins return it (or wrap it) to choose the code;
// any other error becomes CodeInternal.
type Error struct {
Code ErrorCode `json:"code"`
Message string `json:"message"`
Retryable bool `json:"retryable,omitempty"`
Details *ErrorDetails `json:"details,omitempty"`
}
// ErrorDetails refines some codes.
type ErrorDetails struct {
// Fields are per-field problems for CodeInvalidConfig, keyed by the
// dotted field path.
Fields map[string]string `json:"fields,omitempty"`
// RetryAfter is seconds to wait for CodeRateLimited.
RetryAfter int `json:"retryAfter,omitempty"`
}
// ErrorBody is the JSON body of a failed call.
type ErrorBody struct {
Error Error `json:"error"`
}
func (e *Error) Error() string { return fmt.Sprintf("%s: %s", e.Code, e.Message) }
// Errorf builds an Error. Unavailable and rate-limited errors are retryable.
func Errorf(code ErrorCode, format string, args ...any) *Error {
return &Error{
Code: code, Message: fmt.Sprintf(format, args...),
Retryable: code == CodeUnavailable || code == CodeRateLimited,
}
}
// InvalidConfig reports field problems, keyed by dotted path.
func InvalidConfig(message string, fields map[string]string) *Error {
return &Error{Code: CodeInvalidConfig, Message: message, Details: &ErrorDetails{Fields: fields}}
}
// AsError finds an *Error in err's chain.
func AsError(err error) (*Error, bool) {
var e *Error
ok := errors.As(err, &e)
return e, ok
}
// HTTPStatus is the status an error code travels with.
func (c ErrorCode) HTTPStatus() int {
switch c {
case CodeInvalidConfig, CodeBadRequest:
return 400
case CodeUnauthorized:
return 401
case CodeNotFound:
return 404
case CodeRateLimited:
return 429
case CodeUnavailable:
return 503
default:
return 500
}
}

View File

@ -0,0 +1,63 @@
package pluginapi
import (
"fmt"
"strings"
)
// Environment a host plugin is started with.
const (
// EnvSocket is where the plugin must listen (a unix socket path, or a
// host:port when EnvNetwork is "tcp").
EnvSocket = "WEKNORA_PLUGIN_SOCKET"
// EnvNetwork is "unix" (default) or "tcp".
EnvNetwork = "WEKNORA_PLUGIN_NETWORK"
// EnvToken is the bearer token every call from the host carries.
EnvToken = "WEKNORA_PLUGIN_TOKEN"
// EnvPluginID and EnvPluginVersion name the package being run.
EnvPluginID = "WEKNORA_PLUGIN_ID"
EnvPluginVersion = "WEKNORA_PLUGIN_VERSION"
// EnvAddr and EnvSecret configure a remote plugin (not started by a
// host): where to listen and the shared secret calls are signed with.
EnvAddr = "WEKNORA_PLUGIN_ADDR"
EnvSecret = "WEKNORA_PLUGIN_SECRET"
)
// handshakePrefix starts the line a host plugin prints on stdout once it is
// ready: WEKNORA_PLUGIN|1|unix|/path/to.sock
const handshakePrefix = "WEKNORA_PLUGIN"
// Handshake is the readiness line of a host plugin.
type Handshake struct {
Protocol string
Network string // unix | tcp
Address string
}
// String renders the handshake line (without a newline).
func (h Handshake) String() string {
return strings.Join([]string{handshakePrefix, h.Protocol, h.Network, h.Address}, "|")
}
// ParseHandshake reads a handshake line. ok is false for any other line,
// which the host treats as log output.
func ParseHandshake(line string) (h Handshake, ok bool, err error) {
parts := strings.Split(strings.TrimSpace(line), "|")
if len(parts) == 0 || parts[0] != handshakePrefix {
return Handshake{}, false, nil
}
if len(parts) != 4 {
return Handshake{}, true, fmt.Errorf("malformed handshake %q", line)
}
h = Handshake{Protocol: parts[1], Network: parts[2], Address: parts[3]}
if h.Protocol != ProtocolVersion {
return h, true, fmt.Errorf("plugin speaks protocol %s, host speaks %s", h.Protocol, ProtocolVersion)
}
if h.Network != "unix" && h.Network != "tcp" {
return h, true, fmt.Errorf("unknown handshake network %q", h.Network)
}
if h.Address == "" {
return h, true, fmt.Errorf("handshake has no address")
}
return h, true, nil
}

View File

@ -0,0 +1,324 @@
openapi: 3.1.0
info:
title: WeKnora extension protocol
version: "1"
description: |
What WeKnora calls on a code plugin. Every POST takes an Envelope and
answers {"output": ...}, or streams NDJSON Events. Failures answer a
non-2xx status with an ErrorBody. Within a major version the protocol
only grows; both sides ignore fields they do not know.
Host plugins (started by the WeKnora plugin host) are called over the
socket from their handshake with `Authorization: Bearer <token>`. Remote
plugins are called over HTTPS and verify `X-WeKnora-Signature`:
hex HMAC-SHA256 of "<X-WeKnora-Timestamp>.<body>" with the shared secret,
rejecting timestamps more than 5 minutes off.
servers:
- url: http://plugin
security:
- hostToken: []
- signature: []
paths:
/v1/manifest:
get:
summary: What this process is and serves
responses:
"200":
description: The manifest
content:
application/json:
schema: { $ref: "#/components/schemas/Manifest" }
default: { $ref: "#/components/responses/Error" }
/v1/health:
get:
summary: Liveness
responses:
"200":
description: Healthy
content:
application/json:
schema:
type: object
required: [status]
properties: { status: { const: ok } }
default: { $ref: "#/components/responses/Error" }
/v1/config/validate:
post:
summary: Check the plugin's system and tenant configuration
requestBody: { $ref: "#/components/requestBodies/Envelope" }
responses:
"200": { $ref: "#/components/responses/EmptyOutput" }
default: { $ref: "#/components/responses/Error" }
/v1/websearch/{id}/search:
post:
summary: Web search (WebSearchProvider.Search)
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
requestBody:
required: true
content:
application/json:
schema:
allOf:
- $ref: "#/components/schemas/Envelope"
- properties: { input: { $ref: "#/components/schemas/SearchInput" } }
responses:
"200":
description: Results
content:
application/json:
schema:
type: object
required: [output]
properties: { output: { $ref: "#/components/schemas/SearchOutput" } }
default: { $ref: "#/components/responses/Error" }
/v1/connectors/{id}/validate:
post:
summary: Check a data source configuration (Connector.Validate)
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
requestBody: { $ref: "#/components/requestBodies/Envelope" }
responses:
"200": { $ref: "#/components/responses/EmptyOutput" }
default: { $ref: "#/components/responses/Error" }
/v1/connectors/{id}/list-resources:
post:
summary: List syncable resources under a parent (Connector.ListResources)
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
requestBody:
required: true
content:
application/json:
schema:
allOf:
- $ref: "#/components/schemas/Envelope"
- properties:
input:
type: object
properties: { parentId: { type: string } }
responses:
"200":
description: Resources
content:
application/json:
schema:
type: object
required: [output]
properties:
output:
type: object
required: [resources]
properties:
resources: { type: array, items: { $ref: "#/components/schemas/Resource" } }
default: { $ref: "#/components/responses/Error" }
/v1/connectors/{id}/resolve-ancestors:
post:
summary: Ancestors a lazy picker must expand to show a selection
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
requestBody:
required: true
content:
application/json:
schema:
allOf:
- $ref: "#/components/schemas/Envelope"
- properties:
input:
type: object
properties: { resourceIds: { type: array, items: { type: string } } }
responses:
"200":
description: Ancestor IDs
content:
application/json:
schema:
type: object
required: [output]
properties:
output:
type: object
required: [ancestors]
properties: { ancestors: { type: array, items: { type: string } } }
default: { $ref: "#/components/responses/Error" }
/v1/connectors/{id}/fetch:
post:
summary: Stream a full or incremental sync
description: |
Answers NDJSON: "item" events carry a FetchedItem, "checkpoint"
events a Cursor (a complete, resumable snapshot), and the stream
ends with "end" (data = final Cursor) or "error". A stream that
stops without either is treated as interrupted.
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
requestBody:
required: true
content:
application/json:
schema:
allOf:
- $ref: "#/components/schemas/Envelope"
- properties: { input: { $ref: "#/components/schemas/FetchInput" } }
responses:
"200":
description: Event stream
content:
application/x-ndjson:
schema: { $ref: "#/components/schemas/Event" }
default: { $ref: "#/components/responses/Error" }
components:
securitySchemes:
hostToken: { type: http, scheme: bearer }
signature: { type: apiKey, in: header, name: X-WeKnora-Signature }
parameters:
ContributionID:
name: id
in: path
required: true
description: The contribution's local ID from plugin.yaml
schema: { type: string }
requestBodies:
Envelope:
required: true
content:
application/json:
schema: { $ref: "#/components/schemas/Envelope" }
responses:
EmptyOutput:
description: Accepted
content:
application/json:
schema:
type: object
required: [output]
properties: { output: { type: object } }
Error:
description: A failed call
content:
application/json:
schema: { $ref: "#/components/schemas/ErrorBody" }
schemas:
Envelope:
type: object
required: [context, config]
properties:
context:
type: object
required: [tenantId]
properties:
tenantId: { type: integer }
userId: { type: string }
locale: { type: string }
requestId: { type: string }
deadline: { type: string, format: date-time }
host:
type: object
description: Host API access; absent without granted scopes
properties:
url: { type: string }
token: { type: string }
config:
type: object
description: Configuration scopes, secrets decrypted, never merged
properties:
system: { type: object }
tenant: { type: object }
instance: { type: object }
input: {}
Manifest:
type: object
required: [id, version, apiVersion, contributes]
properties:
id: { type: string }
version: { type: string }
apiVersion: { const: weknora.plugin/v1 }
contributes:
type: object
additionalProperties: { type: array, items: { type: string } }
SearchInput:
type: object
required: [query]
properties:
query: { type: string }
maxResults: { type: integer }
includeDate: { type: boolean }
region: { type: string }
freshness: { type: string }
SearchOutput:
type: object
required: [results]
properties:
results:
type: array
items:
type: object
required: [title, url]
properties:
title: { type: string }
url: { type: string }
snippet: { type: string }
content: { type: string }
age: { type: string }
publishedAt: { type: string, format: date-time }
Resource:
type: object
required: [externalId, name]
properties:
externalId: { type: string }
name: { type: string }
type: { type: string }
description: { type: string }
url: { type: string }
modifiedAt: { type: string, format: date-time }
parentId: { type: string }
hasChildren: { type: boolean }
metadata: { type: object }
FetchInput:
type: object
required: [mode]
properties:
mode: { enum: [full, incremental] }
cursor: { $ref: "#/components/schemas/Cursor" }
resourceIds: { type: array, items: { type: string } }
Cursor:
type: object
properties:
lastSyncTime: { type: string, format: date-time }
state: { type: object }
FetchedItem:
type: object
required: [externalId, title]
properties:
externalId: { type: string }
title: { type: string }
content: { type: string, contentEncoding: base64 }
contentType: { type: string }
fileName: { type: string }
url: { type: string }
updatedAt: { type: string, format: date-time }
createdAt: { type: string, format: date-time }
metadata: { type: object, additionalProperties: { type: string } }
isDeleted: { type: boolean }
sourceResourceId: { type: string }
Event:
type: object
required: [type]
properties:
type: { enum: [item, checkpoint, progress, log, error, end] }
data: {}
error: { $ref: "#/components/schemas/Error" }
message: { type: string }
level: { enum: [debug, info, warn, error] }
Error:
type: object
required: [code, message]
properties:
code: { enum: [invalid_config, unauthorized, not_found, rate_limited, unavailable, bad_request, internal] }
message: { type: string }
retryable: { type: boolean }
details:
type: object
properties:
fields: { type: object, additionalProperties: { type: string } }
retryAfter: { type: integer }
ErrorBody:
type: object
required: [error]
properties:
error: { $ref: "#/components/schemas/Error" }

View File

@ -0,0 +1,48 @@
package pluginapi
import (
"strconv"
"testing"
"time"
)
func TestHandshake(t *testing.T) {
h := Handshake{Protocol: ProtocolVersion, Network: "unix", Address: "/tmp/p.sock"}
got, ok, err := ParseHandshake(h.String() + "\n")
if !ok || err != nil || got != h {
t.Fatalf("round trip = %+v %v %v", got, ok, err)
}
if _, ok, _ := ParseHandshake("starting up..."); ok {
t.Fatal("other output is not a handshake")
}
bad := []string{
"WEKNORA_PLUGIN|2|unix|/x", "WEKNORA_PLUGIN|1|pipe|/x", "WEKNORA_PLUGIN|1|unix|", "WEKNORA_PLUGIN|1",
}
for _, bad := range bad {
if _, ok, err := ParseHandshake(bad); !ok || err == nil {
t.Errorf("%q must be a rejected handshake", bad)
}
}
}
func TestSignature(t *testing.T) {
secret, body, now := []byte("k"), []byte(`{"a":1}`), time.Now()
ts := strconv.FormatInt(now.Unix(), 10)
sig := Sign(secret, now.Unix(), body)
if err := VerifySignature(secret, ts, sig, body, now); err != nil {
t.Fatal(err)
}
if VerifySignature(secret, ts, sig, []byte(`{"a":2}`), now) == nil {
t.Fatal("a changed body must fail")
}
if VerifySignature(secret, ts, sig, body, now.Add(10*time.Minute)) == nil {
t.Fatal("a replayed old request must fail")
}
}
func TestErrorStatus(t *testing.T) {
if Errorf(CodeRateLimited, "x").Code.HTTPStatus() != 429 || !Errorf(CodeUnavailable, "x").Retryable ||
Errorf(CodeInternal, "x").Retryable {
t.Fatal("error codes map wrong")
}
}

View File

@ -0,0 +1,37 @@
package pluginapi
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"strconv"
"time"
)
// MaxClockSkew bounds how old a signed request may be.
const MaxClockSkew = 5 * time.Minute
// Sign returns the signature of a request body at a unix timestamp.
func Sign(secret []byte, timestamp int64, body []byte) string {
mac := hmac.New(sha256.New, secret)
_, _ = fmt.Fprintf(mac, "%d.", timestamp)
_, _ = mac.Write(body)
return hex.EncodeToString(mac.Sum(nil))
}
// VerifySignature checks a signed request against the secret and clock.
func VerifySignature(secret []byte, timestamp, signature string, body []byte, now time.Time) error {
ts, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return fmt.Errorf("bad %s", TimestampHeader)
}
if d := now.Sub(time.Unix(ts, 0)); d > MaxClockSkew || d < -MaxClockSkew {
return fmt.Errorf("request timestamp is outside the allowed clock skew")
}
want := Sign(secret, ts, body)
if !hmac.Equal([]byte(want), []byte(signature)) {
return fmt.Errorf("bad signature")
}
return nil
}

View File

@ -0,0 +1,34 @@
package pluginapi
import "encoding/json"
// EventType is the kind of one NDJSON line in a streaming answer.
type EventType string
const (
// EventItem carries one result (a FetchedItem for connectors).
EventItem EventType = "item"
// EventCheckpoint carries a resumable cursor; WeKnora persists it.
EventCheckpoint EventType = "checkpoint"
// EventProgress reports progress for display.
EventProgress EventType = "progress"
// EventLog forwards a log line to WeKnora's logs.
EventLog EventType = "log"
// EventError ends the stream with an error.
EventError EventType = "error"
// EventEnd ends the stream successfully, with the final Data.
EventEnd EventType = "end"
)
// Event is one line of a streaming answer. A stream that stops without an
// "end" or "error" event was interrupted and is treated as failed.
type Event struct {
Type EventType `json:"type"`
Data json.RawMessage `json:"data,omitempty"`
// Error is set on EventError.
Error *Error `json:"error,omitempty"`
// Message is set on EventLog and EventProgress.
Message string `json:"message,omitempty"`
// Level is set on EventLog: debug, info, warn, error.
Level string `json:"level,omitempty"`
}

View File

@ -0,0 +1,32 @@
package pluginapi
import "time"
// SearchPath is the web search endpoint for provider id.
func SearchPath(id string) string { return "/v1/websearch/" + id + "/search" }
// SearchInput is the input of a web search.
type SearchInput struct {
Query string `json:"query"`
MaxResults int `json:"maxResults"`
IncludeDate bool `json:"includeDate,omitempty"`
// Region and Freshness are optional filters; a provider that cannot
// honour a requested filter must fail rather than ignore it.
Region string `json:"region,omitempty"`
Freshness string `json:"freshness,omitempty"`
}
// SearchOutput is the output of a web search.
type SearchOutput struct {
Results []SearchResult `json:"results"`
}
// SearchResult is one hit.
type SearchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet,omitempty"`
Content string `json:"content,omitempty"`
Age string `json:"age,omitempty"`
PublishedAt *time.Time `json:"publishedAt,omitempty"`
}

130
pluginsdk/stream.go 100644
View File

@ -0,0 +1,130 @@
package pluginsdk
import (
"encoding/json"
"net/http"
"sync"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Stream writes a streaming answer: items, checkpoints and progress as NDJSON
// lines, flushed one by one. It is safe for concurrent use.
type Stream struct {
mu sync.Mutex
w http.ResponseWriter
flusher http.Flusher
started bool
closed bool
err error
}
func newStream(w http.ResponseWriter) *Stream {
f, _ := w.(http.Flusher)
return &Stream{w: w, flusher: f}
}
func (s *Stream) write(ev pluginapi.Event) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.err != nil {
return s.err
}
if s.closed {
return pluginapi.Errorf(pluginapi.CodeInternal, "stream already ended")
}
if !s.started {
s.w.Header().Set("Content-Type", pluginapi.NDJSONContentType)
s.w.WriteHeader(http.StatusOK)
s.started = true
}
b, err := json.Marshal(ev)
if err != nil {
return err
}
if _, err := s.w.Write(append(b, '\n')); err != nil {
// The caller went away; stop producing.
s.err = err
return err
}
if s.flusher != nil {
s.flusher.Flush()
}
return nil
}
func dataEvent(t pluginapi.EventType, v any) (pluginapi.Event, error) {
raw, err := json.Marshal(v)
if err != nil {
return pluginapi.Event{}, err
}
return pluginapi.Event{Type: t, Data: raw}, nil
}
// Item emits one fetched item. An error means WeKnora stopped listening:
// stop fetching and return.
func (s *Stream) Item(item pluginapi.FetchedItem) error {
ev, err := dataEvent(pluginapi.EventItem, item)
if err != nil {
return err
}
return s.write(ev)
}
// Checkpoint emits a resumable cursor. It must be a complete snapshot.
func (s *Stream) Checkpoint(c pluginapi.Cursor) error {
ev, err := dataEvent(pluginapi.EventCheckpoint, c)
if err != nil {
return err
}
return s.write(ev)
}
// Progress reports progress for display.
func (s *Stream) Progress(message string) error {
return s.write(pluginapi.Event{Type: pluginapi.EventProgress, Message: message})
}
// Log forwards a line to WeKnora's logs (level: debug, info, warn, error).
func (s *Stream) Log(level, message string) error {
return s.write(pluginapi.Event{Type: pluginapi.EventLog, Level: level, Message: message})
}
func (s *Stream) end(cursor *pluginapi.Cursor) {
if cursor == nil {
cursor = &pluginapi.Cursor{}
}
ev, err := dataEvent(pluginapi.EventEnd, cursor)
if err == nil {
err = s.write(ev)
}
if err != nil {
s.fail(err)
return
}
s.mu.Lock()
s.closed = true
s.mu.Unlock()
}
// fail ends the stream with an error: as an error event once lines went
// out, or as an ordinary error answer before that.
func (s *Stream) fail(err error) {
s.mu.Lock()
started, closed := s.started, s.closed
s.mu.Unlock()
if closed {
return
}
if !started {
writeError(s.w, err)
s.mu.Lock()
s.closed = true
s.mu.Unlock()
return
}
_ = s.write(pluginapi.Event{Type: pluginapi.EventError, Error: toError(err)})
s.mu.Lock()
s.closed = true
s.mu.Unlock()
}

View File

@ -0,0 +1,57 @@
package pluginsdk
import (
"context"
"encoding/json"
"net/http"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// WebSearcher is a web search provider. Its instance configuration is what
// the contribution's instanceSchema describes.
type WebSearcher interface {
Search(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error)
}
// WebSearchFunc adapts a function to WebSearcher.
type WebSearchFunc func(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error)
// Search implements WebSearcher.
func (f WebSearchFunc) Search(
ctx context.Context,
call *Call,
in pluginapi.SearchInput,
) (*pluginapi.SearchOutput, error) {
return f(ctx, call, in)
}
// WebSearch registers web search provider id (contributes.webSearch[].id).
func (p *Plugin) WebSearch(id string, s WebSearcher) { p.webSearch[id] = s }
func (p *Plugin) routeWebSearch(mux *http.ServeMux) {
mux.HandleFunc("POST /v1/websearch/{id}/search", func(w http.ResponseWriter, r *http.Request) {
s, ok := p.webSearch[r.PathValue("id")]
if !ok {
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no web search provider %q", r.PathValue("id")))
return
}
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
in, err := decodeInput[pluginapi.SearchInput](raw)
if err != nil {
return nil, err
}
out, err := s.Search(ctx, call, in)
if err != nil {
return nil, err
}
if out == nil {
out = &pluginapi.SearchOutput{}
}
if out.Results == nil {
out.Results = []pluginapi.SearchResult{}
}
return out, nil
})(w, r)
})
}