feat(plugin): code plugins — extension protocol, Go SDK, plugin host, web search and data source adapters (#3720)
* feat(pluginsdk): extension protocol v1, Go SDK and conformance suite * feat(plugin): run code plugins in an embedded plugin host * feat(plugin): remote adapters for web search and data sources * fix(plugin): show plugin errors by their message * feat(frontend): plugin connectors and web search providers in the UI * docs(pluginsdk): how to write, package, test and install a plugin * fix(plugin): gofmt two files and read openapi.yaml line-ending agnostic * fix(plugin): mark a host plugin ready before Activate returnsmain
parent
ed63e35cc7
commit
4328ab26a9
|
|
@ -58,10 +58,10 @@ jobs:
|
|||
modules=()
|
||||
if git diff --name-only --diff-filter=ACMR "$diff_range" -- \
|
||||
.golangci.yml .github/workflows/go-lint.yml | grep -q .; then
|
||||
modules=("." "cli" "client")
|
||||
modules=("." "cli" "client" "pluginsdk")
|
||||
else
|
||||
if git diff --name-only --diff-filter=ACMR "$diff_range" -- \
|
||||
'*.go' ':!cli/**' ':!client/**' go.mod go.sum | grep -q .; then
|
||||
'*.go' ':!cli/**' ':!client/**' ':!pluginsdk/**' go.mod go.sum | grep -q .; then
|
||||
modules+=(".")
|
||||
fi
|
||||
if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'cli/**' | grep -q .; then
|
||||
|
|
@ -70,6 +70,9 @@ jobs:
|
|||
if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'client/**' | grep -q .; then
|
||||
modules+=("client")
|
||||
fi
|
||||
if git diff --name-only --diff-filter=ACMR "$diff_range" -- 'pluginsdk/**' | grep -q .; then
|
||||
modules+=("pluginsdk")
|
||||
fi
|
||||
fi
|
||||
json="["
|
||||
first=1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
name: Plugin SDK
|
||||
|
||||
# Builds and tests the pluginsdk/ Go module (extension protocol, Go SDK,
|
||||
# client and conformance suite) on every push / PR that touches it.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'pluginsdk/**'
|
||||
- '.github/workflows/plugin-sdk.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'pluginsdk/**'
|
||||
- '.github/workflows/plugin-sdk.yml'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: pluginsdk
|
||||
shell: bash
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.26'
|
||||
cache-dependency-path: pluginsdk/go.mod
|
||||
- name: go build
|
||||
run: go build ./...
|
||||
- name: go test
|
||||
run: go test -race ./...
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
|
@ -38,10 +38,12 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
|
|||
# Install migrate tool
|
||||
RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
|
||||
|
||||
# Copy go mod files. go.mod replace-points anydoc at ./third_party/anydoc-go,
|
||||
# so that module's go.mod must exist before `go mod download`.
|
||||
# Copy go mod files. go.mod replace-points anydoc at ./third_party/anydoc-go
|
||||
# and the plugin SDK at ./pluginsdk, so their go.mod files must exist before
|
||||
# `go mod download`.
|
||||
COPY go.mod go.sum ./
|
||||
COPY third_party/anydoc-go/go.mod third_party/anydoc-go/go.mod
|
||||
COPY pluginsdk/go.mod pluginsdk/go.mod
|
||||
RUN --mount=type=cache,target=/go/pkg/mod go mod download
|
||||
COPY cmd/download cmd/download
|
||||
RUN go run cmd/download/duckdb/duckdb.go
|
||||
|
|
|
|||
|
|
@ -0,0 +1,282 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
readability "codeberg.org/readeck/go-readability/v2"
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
const (
|
||||
requestTimeout = 20 * time.Second
|
||||
maxFeedSize = 10 << 20
|
||||
maxArticleSize = 5 << 20
|
||||
defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
|
||||
maxFileNameBytes = 200
|
||||
)
|
||||
|
||||
// config is the data source configuration: feed URLs in settings, optional
|
||||
// auth headers (which may carry secrets) in credentials.
|
||||
type config struct {
|
||||
feedURLs []string
|
||||
authHeaders map[string]string
|
||||
}
|
||||
|
||||
func parseConfig(settings, credentials map[string]any) (*config, error) {
|
||||
raw, _ := settings["feed_urls"].(string)
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
// Older rows kept the URLs with the credentials.
|
||||
raw, _ = credentials["feed_urls"].(string)
|
||||
}
|
||||
cfg := &config{feedURLs: splitURLs(raw)}
|
||||
if len(cfg.feedURLs) == 0 {
|
||||
return nil, errMissingFeeds
|
||||
}
|
||||
headers, _ := credentials["auth_headers"].(string)
|
||||
cfg.authHeaders = parseHeaders(headers)
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// splitURLs splits on newlines and commas, trims, dedupes (order kept).
|
||||
func splitURLs(raw string) []string {
|
||||
parts := strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' })
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, u := range parts {
|
||||
u = strings.TrimSpace(u)
|
||||
if u == "" || seen[u] {
|
||||
continue
|
||||
}
|
||||
seen[u] = true
|
||||
out = append(out, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseHeaders(raw string) map[string]string {
|
||||
headers := map[string]string{}
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
name, value, ok := strings.Cut(strings.TrimSpace(line), ":")
|
||||
name = strings.TrimSpace(name)
|
||||
if !ok || name == "" {
|
||||
continue
|
||||
}
|
||||
headers[name] = strings.TrimSpace(value)
|
||||
}
|
||||
if len(headers) == 0 {
|
||||
return nil
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// fetcher downloads feeds and article pages. Outbound traffic goes through
|
||||
// the plugin host's egress proxy (HTTPS_PROXY), which refuses private
|
||||
// addresses, so the plugin needs no SSRF logic of its own.
|
||||
type fetcher struct {
|
||||
http *http.Client
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
func newFetcher(headers map[string]string) *fetcher {
|
||||
return &fetcher{http: &http.Client{Timeout: requestTimeout}, headers: headers}
|
||||
}
|
||||
|
||||
// fetch attaches auth headers only to feed fetches; article pages on
|
||||
// third-party sites must not receive feed credentials.
|
||||
func (f *fetcher) fetch(ctx context.Context, rawURL string, maxSize int64, withAuth bool) ([]byte, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return nil, fmt.Errorf("invalid URL %q", rawURL)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if withAuth {
|
||||
for k, v := range f.headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
if req.Header.Get("Accept") == "" {
|
||||
req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, "+
|
||||
"application/json, text/html;q=0.9, */*;q=0.8")
|
||||
}
|
||||
resp, err := f.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body failed: %w", err)
|
||||
}
|
||||
if int64(len(body)) > maxSize {
|
||||
return nil, fmt.Errorf("response exceeds maximum size (%d bytes)", maxSize)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (f *fetcher) feed(ctx context.Context, feedURL string) (*gofeed.Feed, error) {
|
||||
data, err := f.fetch(ctx, feedURL, maxFeedSize, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
feed, err := gofeed.NewParser().Parse(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse feed: %w", err)
|
||||
}
|
||||
return feed, nil
|
||||
}
|
||||
|
||||
// article returns the readability-cleaned main content of a page as HTML.
|
||||
func (f *fetcher) article(ctx context.Context, articleURL string) (string, string, error) {
|
||||
body, err := f.fetch(ctx, articleURL, maxArticleSize, false)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
pageURL, _ := url.Parse(articleURL)
|
||||
art, err := readability.FromReader(bytes.NewReader(body), pageURL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("readability parse: %w", err)
|
||||
}
|
||||
if art.Node == nil {
|
||||
return "", "", fmt.Errorf("no readable content extracted")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := art.RenderHTML(&buf); err != nil {
|
||||
return "", "", fmt.Errorf("render article html: %w", err)
|
||||
}
|
||||
return buf.String(), art.Title(), nil
|
||||
}
|
||||
|
||||
// cursorState is the incremental sync state, the same shape the builtin
|
||||
// connector keeps: feed URL → item ID → content fingerprint, and feed URL →
|
||||
// item ID → feed-only signal that lets unchanged entries skip article fetches.
|
||||
type cursorState struct {
|
||||
FeedItems map[string]map[string]string
|
||||
FeedSignals map[string]map[string]string
|
||||
}
|
||||
|
||||
func newCursorState() *cursorState {
|
||||
return &cursorState{FeedItems: map[string]map[string]string{}, FeedSignals: map[string]map[string]string{}}
|
||||
}
|
||||
|
||||
func cursorFromState(state map[string]any) *cursorState {
|
||||
c := newCursorState()
|
||||
read := func(key string, into map[string]map[string]string) {
|
||||
feeds, _ := state[key].(map[string]any)
|
||||
for feedURL, items := range feeds {
|
||||
m, _ := items.(map[string]any)
|
||||
into[feedURL] = map[string]string{}
|
||||
for id, v := range m {
|
||||
if s, ok := v.(string); ok {
|
||||
into[feedURL][id] = s
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
read("feed_items", c.FeedItems)
|
||||
read("feed_signals", c.FeedSignals)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *cursorState) toState() map[string]any {
|
||||
return map[string]any{"feed_items": c.FeedItems, "feed_signals": c.FeedSignals}
|
||||
}
|
||||
|
||||
func (c *cursorState) copyFeed(prev *cursorState, feedURL string) {
|
||||
if prev == nil {
|
||||
return
|
||||
}
|
||||
if src := prev.FeedItems[feedURL]; len(src) > 0 {
|
||||
c.FeedItems[feedURL] = maps.Clone(src)
|
||||
}
|
||||
if src := prev.FeedSignals[feedURL]; len(src) > 0 {
|
||||
c.FeedSignals[feedURL] = maps.Clone(src)
|
||||
}
|
||||
}
|
||||
|
||||
func contentFingerprint(markdown string) string {
|
||||
sum := sha256.Sum256([]byte(markdown))
|
||||
return "h:" + hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
func feedSignal(item *gofeed.Item, feedContent string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(item.GUID + "\n" + item.Link + "\n" + item.Title + "\n")
|
||||
if item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero() {
|
||||
b.WriteString(item.UpdatedParsed.UTC().Format(time.RFC3339))
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
if item.PublishedParsed != nil && !item.PublishedParsed.IsZero() {
|
||||
b.WriteString(item.PublishedParsed.UTC().Format(time.RFC3339))
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
b.WriteString(feedContent)
|
||||
sum := sha256.Sum256([]byte(b.String()))
|
||||
return "s:" + hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func htmlToMarkdown(html string) string {
|
||||
if strings.TrimSpace(html) == "" {
|
||||
return ""
|
||||
}
|
||||
md, err := htmltomd.ConvertString(html)
|
||||
if err != nil || strings.TrimSpace(md) == "" {
|
||||
return strings.TrimSpace(html)
|
||||
}
|
||||
return strings.TrimSpace(md)
|
||||
}
|
||||
|
||||
var fileNameReplacer = strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
|
||||
// fileName makes a feed title safe to use as a file name, the way WeKnora's
|
||||
// builtin connectors do: hostile punctuation replaced, at most 200 bytes
|
||||
// without splitting a rune.
|
||||
func fileName(title string) string {
|
||||
title = strings.TrimSpace(strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(title))
|
||||
if title == "" {
|
||||
return "untitled.md"
|
||||
}
|
||||
name := fileNameReplacer.Replace(title)
|
||||
if len(name) > maxFileNameBytes {
|
||||
name = name[:maxFileNameBytes]
|
||||
for len(name) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(name)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
name = name[:len(name)-1]
|
||||
}
|
||||
}
|
||||
return name + ".md"
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><rect width="24" height="24" rx="5" fill="#f26522"/><circle cx="6.5" cy="17.5" r="2" fill="#fff"/><path d="M4.5 10.5a9 9 0 0 1 9 9h-2.6a6.4 6.4 0 0 0-6.4-6.4z" fill="#fff"/><path d="M4.5 4.5a15 15 0 0 1 15 15h-2.6A12.4 12.4 0 0 0 4.5 7.1z" fill="#fff"/></svg>
|
||||
|
After Width: | Height: | Size: 320 B |
|
|
@ -0,0 +1,229 @@
|
|||
// Command rss is WeKnora's RSS/Atom/JSON feed connector written as an
|
||||
// external plugin with the Go SDK. It behaves like the builtin rss connector
|
||||
// (a test syncs the same feeds through both) and shows how a code plugin is
|
||||
// built: it imports only the SDK and ordinary libraries, never WeKnora's
|
||||
// internals.
|
||||
//
|
||||
// Build a package with ./package.sh; install it under System administration
|
||||
// → Plugin management.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Version must match plugin.yaml.
|
||||
const Version = "1.0.0"
|
||||
|
||||
var errMissingFeeds = pluginapi.InvalidConfig(
|
||||
"feed_urls is required",
|
||||
map[string]string{"settings.feed_urls": "required"},
|
||||
)
|
||||
|
||||
func main() {
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version})
|
||||
p.Connector("rss", &connector{logf: p.Logger().Warn})
|
||||
if err := p.Serve(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type connector struct {
|
||||
logf func(msg string, args ...any)
|
||||
}
|
||||
|
||||
func (c *connector) warn(msg string, args ...any) {
|
||||
if c.logf != nil {
|
||||
c.logf(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate fetches and parses every configured feed.
|
||||
func (c *connector) Validate(ctx context.Context, _ *pluginsdk.Call, in pluginsdk.ConnectorConfig) error {
|
||||
cfg, err := parseConfig(in.Settings, in.Credentials)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := newFetcher(cfg.authHeaders)
|
||||
for _, u := range cfg.feedURLs {
|
||||
if _, err := f.feed(ctx, u); err != nil {
|
||||
return pluginapi.InvalidConfig(fmt.Sprintf("feed %s: %v", u, err),
|
||||
map[string]string{"settings.feed_urls": err.Error()})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListResources lists one resource per feed. A feed that fails still
|
||||
// appears, named by its URL, so the user can deselect it.
|
||||
func (c *connector) ListResources(
|
||||
ctx context.Context, _ *pluginsdk.Call, in pluginsdk.ConnectorConfig, parentID string,
|
||||
) ([]pluginapi.Resource, error) {
|
||||
if parentID != "" {
|
||||
return nil, nil
|
||||
}
|
||||
cfg, err := parseConfig(in.Settings, in.Credentials)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := newFetcher(cfg.authHeaders)
|
||||
out := make([]pluginapi.Resource, 0, len(cfg.feedURLs))
|
||||
for _, u := range cfg.feedURLs {
|
||||
res := pluginapi.Resource{ExternalID: u, Type: "feed", Name: u, URL: u}
|
||||
feed, err := f.feed(ctx, u)
|
||||
if err != nil {
|
||||
res.Description = "fetch failed: " + err.Error()
|
||||
out = append(out, res)
|
||||
continue
|
||||
}
|
||||
if t := strings.TrimSpace(feed.Title); t != "" {
|
||||
res.Name = t
|
||||
}
|
||||
res.Description = strings.TrimSpace(feed.Description)
|
||||
if feed.Link != "" {
|
||||
res.URL = feed.Link
|
||||
}
|
||||
res.ModifiedAt = feed.UpdatedParsed
|
||||
res.Metadata = map[string]any{"item_count": len(feed.Items)}
|
||||
out = append(out, res)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Fetch syncs the selected feeds (all configured ones when none are
|
||||
// selected). Incremental runs skip entries whose feed signal and content are
|
||||
// unchanged; deletions are never emitted, since feeds drop old items as a
|
||||
// matter of course.
|
||||
func (c *connector) Fetch(
|
||||
ctx context.Context,
|
||||
_ *pluginsdk.Call,
|
||||
in pluginsdk.ConnectorConfig,
|
||||
req pluginapi.FetchInput,
|
||||
s *pluginsdk.Stream,
|
||||
) (*pluginapi.Cursor, error) {
|
||||
cfg, err := parseConfig(in.Settings, in.Credentials)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
feedURLs := req.ResourceIDs
|
||||
if len(feedURLs) == 0 {
|
||||
feedURLs = cfg.feedURLs
|
||||
}
|
||||
var prev *cursorState
|
||||
incremental := req.Mode == pluginapi.FetchIncremental && req.Cursor != nil
|
||||
if incremental {
|
||||
prev = cursorFromState(req.Cursor.State)
|
||||
}
|
||||
f := newFetcher(cfg.authHeaders)
|
||||
next := newCursorState()
|
||||
now := time.Now().UTC()
|
||||
var failures []string
|
||||
|
||||
for _, feedURL := range feedURLs {
|
||||
feed, err := f.feed(ctx, feedURL)
|
||||
if err != nil {
|
||||
c.warn("feed failed", "url", feedURL, "err", err)
|
||||
failures = append(failures, fmt.Sprintf("%s: %v", feedURL, err))
|
||||
next.copyFeed(prev, feedURL)
|
||||
continue
|
||||
}
|
||||
next.FeedItems[feedURL] = map[string]string{}
|
||||
next.FeedSignals[feedURL] = map[string]string{}
|
||||
var prevItems, prevSignals map[string]string
|
||||
if prev != nil {
|
||||
prevItems, prevSignals = prev.FeedItems[feedURL], prev.FeedSignals[feedURL]
|
||||
}
|
||||
for _, item := range feed.Items {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
itemID := firstNonEmpty(item.GUID, item.Link, item.Title)
|
||||
if itemID == "" {
|
||||
continue
|
||||
}
|
||||
feedContent := firstNonEmpty(item.Content, item.Description)
|
||||
sig := feedSignal(item, feedContent)
|
||||
if incremental && prevItems[itemID] != "" && prevSignals[itemID] == sig {
|
||||
next.FeedItems[feedURL][itemID] = prevItems[itemID]
|
||||
next.FeedSignals[feedURL][itemID] = sig
|
||||
continue
|
||||
}
|
||||
fetched, fp := c.resolve(ctx, f, feed, item, feedURL, itemID, feedContent)
|
||||
next.FeedItems[feedURL][itemID] = fp
|
||||
next.FeedSignals[feedURL][itemID] = sig
|
||||
if incremental && prevItems[itemID] == fp {
|
||||
continue
|
||||
}
|
||||
if err := s.Item(fetched); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Each finished feed is a safe place to resume from.
|
||||
if err := s.Checkpoint(pluginapi.Cursor{LastSyncTime: &now, State: next.toState()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 && len(failures) == len(feedURLs) {
|
||||
return nil, pluginapi.Errorf(pluginapi.CodeUnavailable, "all feeds failed: %s", strings.Join(failures, "; "))
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
_ = s.Log("warn", "some feeds failed: "+strings.Join(failures, "; "))
|
||||
}
|
||||
return &pluginapi.Cursor{LastSyncTime: &now, State: next.toState()}, nil
|
||||
}
|
||||
|
||||
// resolve builds one item: the article's full text when it can be fetched,
|
||||
// the feed's own content otherwise, as Markdown.
|
||||
func (c *connector) resolve(
|
||||
ctx context.Context, f *fetcher, feed *gofeed.Feed, item *gofeed.Item, feedURL, itemID, feedContent string,
|
||||
) (pluginapi.FetchedItem, string) {
|
||||
title := firstNonEmpty(item.Title, "untitled")
|
||||
contentHTML := feedContent
|
||||
if strings.TrimSpace(item.Link) != "" {
|
||||
html, articleTitle, err := f.article(ctx, item.Link)
|
||||
if err == nil {
|
||||
contentHTML = html
|
||||
if item.Title == "" && articleTitle != "" {
|
||||
title = articleTitle
|
||||
}
|
||||
} else if !errors.Is(err, context.Canceled) {
|
||||
c.warn("full-text fetch failed, using feed content", "url", item.Link, "err", err)
|
||||
}
|
||||
}
|
||||
content := htmlToMarkdown(contentHTML)
|
||||
updated := time.Now().UTC()
|
||||
switch {
|
||||
case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero():
|
||||
updated = *item.UpdatedParsed
|
||||
case item.PublishedParsed != nil && !item.PublishedParsed.IsZero():
|
||||
updated = *item.PublishedParsed
|
||||
}
|
||||
author := ""
|
||||
if item.Author != nil {
|
||||
author = item.Author.Name
|
||||
}
|
||||
return pluginapi.FetchedItem{
|
||||
ExternalID: feedURL + ":" + itemID,
|
||||
Title: title,
|
||||
Content: []byte(content),
|
||||
ContentType: "text/markdown",
|
||||
FileName: fileName(title),
|
||||
URL: item.Link,
|
||||
UpdatedAt: &updated,
|
||||
SourceResourceID: feedURL,
|
||||
Metadata: map[string]string{
|
||||
"channel": "rss", "feed_url": feedURL, "feed_title": feed.Title,
|
||||
"guid": item.GUID, "link": item.Link, "author": author,
|
||||
},
|
||||
}, contentFingerprint(content)
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env bash
|
||||
# Builds the RSS example plugin for the platforms WeKnora runs on and zips it
|
||||
# into a .wkp package: plugin.yaml, schemas, icon and bin/<os>-<arch>/rss.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
version=$(sed -n 's/^version: //p' plugin.yaml)
|
||||
out=${1:-"weknora-examples-rss-${version}.wkp"}
|
||||
stage=$(mktemp -d)
|
||||
trap 'rm -rf "$stage"' EXIT
|
||||
cp plugin.yaml icon.svg "$stage"/
|
||||
cp -r schemas "$stage"/
|
||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64; do
|
||||
os=${target%/*}
|
||||
arch=${target#*/}
|
||||
bin="$stage/bin/${os}-${arch}/rss"
|
||||
[ "$os" = windows ] && bin="$bin.exe"
|
||||
CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -trimpath -ldflags="-s -w" -o "$bin" .
|
||||
done
|
||||
rm -f "$out"
|
||||
(cd "$stage" && zip -qr - .) > "$out"
|
||||
echo "wrote $out"
|
||||
|
|
@ -0,0 +1,279 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
builtin "github.com/Tencent/WeKnora/internal/datasource/connector/rss"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/conformance"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// feedSite serves two feeds and the article pages they link to. Changing an
|
||||
// entry's text simulates a feed update between syncs.
|
||||
type feedSite struct {
|
||||
*httptest.Server
|
||||
mu sync.Mutex
|
||||
secondA string
|
||||
}
|
||||
|
||||
func newFeedSite(t *testing.T) *feedSite {
|
||||
s := &feedSite{secondA: "Second post, first version."}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/a.xml", func(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
second := s.secondA
|
||||
s.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/rss+xml")
|
||||
_, _ = fmt.Fprintf(w, `<?xml version="1.0"?><rss version="2.0"><channel><title>Feed A</title><link>%[1]s</link>
|
||||
<description>First feed</description>
|
||||
<item><title>Hello: world</title><link>%[1]s/articles/1</link><guid>a-1</guid>
|
||||
<pubDate>Mon, 02 Jan 2006 15:04:05 GMT</pubDate><description>Teaser one</description></item>
|
||||
<item><title>Second</title><guid>a-2</guid><pubDate>Tue, 03 Jan 2006 15:04:05 GMT</pubDate>
|
||||
<description><![CDATA[<p>%[2]s</p>]]></description></item>
|
||||
</channel></rss>`, "http://"+r.Host, second)
|
||||
})
|
||||
mux.HandleFunc("/b.atom", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/atom+xml")
|
||||
_, _ = fmt.Fprintf(w, `<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>Feed B</title><id>urn:b</id><updated>2006-01-04T00:00:00Z</updated>
|
||||
<entry><title>Atom entry</title><id>urn:b:1</id><updated>2006-01-04T00:00:00Z</updated>
|
||||
<author><name>Ada</name></author><content type="html"><h2>Atom</h2><p>body</p></content></entry>
|
||||
</feed>`)
|
||||
})
|
||||
mux.HandleFunc("/articles/1", func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = fmt.Fprint(w, `<html><head><title>Article one</title></head><body><nav>menu</nav><article>
|
||||
<h1>Hello world</h1><p>The full text of the first article is long enough for a readability extractor to keep it.
|
||||
It carries several sentences, so the page is recognised as the main content and not as chrome.</p>
|
||||
<p>A second paragraph makes the article clearly the densest block on the page.</p></article></body></html>`)
|
||||
})
|
||||
s.Server = httptest.NewServer(mux)
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
// syncBoth runs the builtin connector and the plugin over the same config.
|
||||
type harness struct {
|
||||
t *testing.T
|
||||
builtin *builtin.Connector
|
||||
plugin *client.Client
|
||||
cfg *types.DataSourceConfig
|
||||
}
|
||||
|
||||
func newHarness(t *testing.T) (*harness, *feedSite) {
|
||||
t.Helper()
|
||||
site := newFeedSite(t)
|
||||
// The builtin connector fetches through the SSRF-safe client.
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||
utils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(utils.ResetSSRFWhitelistForTest)
|
||||
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version})
|
||||
p.Connector("rss", &connector{})
|
||||
srv := httptest.NewServer(p.Handler())
|
||||
t.Cleanup(srv.Close)
|
||||
return &harness{
|
||||
t: t, builtin: builtin.NewConnector(), plugin: client.New(srv.URL, nil, nil),
|
||||
cfg: &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Settings: map[string]any{"feed_urls": site.URL + "/a.xml\n" + site.URL + "/b.atom"},
|
||||
},
|
||||
}, site
|
||||
}
|
||||
|
||||
func (h *harness) pluginFetch(mode string, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor) {
|
||||
h.t.Helper()
|
||||
instance := map[string]any{"settings": h.cfg.Settings, "credentials": map[string]any{}}
|
||||
in := pluginapi.FetchInput{Mode: mode}
|
||||
if cursor != nil {
|
||||
in.Cursor = &pluginapi.Cursor{State: cursor.ConnectorCursor}
|
||||
}
|
||||
var items []types.FetchedItem
|
||||
end, err := h.plugin.Stream(context.Background(), pluginapi.ConnectorFetchPath("rss"),
|
||||
pluginapi.Envelope{Config: pluginapi.Config{Instance: instance}}, in, func(ev pluginapi.Event) error {
|
||||
if ev.Type != pluginapi.EventItem {
|
||||
return nil
|
||||
}
|
||||
var it pluginapi.FetchedItem
|
||||
if err := json.Unmarshal(ev.Data, &it); err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, types.FetchedItem{
|
||||
ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType,
|
||||
FileName: it.FileName, URL: it.URL, UpdatedAt: *it.UpdatedAt, Metadata: it.Metadata,
|
||||
SourceResourceID: it.SourceResourceID,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
h.t.Fatalf("plugin fetch: %v", err)
|
||||
}
|
||||
var c pluginapi.Cursor
|
||||
if err := json.Unmarshal(end, &c); err != nil {
|
||||
h.t.Fatal(err)
|
||||
}
|
||||
return items, &types.SyncCursor{ConnectorCursor: c.State}
|
||||
}
|
||||
|
||||
// comparable renders items in a stable form; UpdatedAt of an undated item is
|
||||
// "now" on both sides, so it is compared only when the feed dated the item.
|
||||
func comparable(items []types.FetchedItem) []string {
|
||||
var out []string
|
||||
for _, it := range items {
|
||||
meta, _ := json.Marshal(it.Metadata)
|
||||
updated := it.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
if time.Since(it.UpdatedAt) < time.Hour {
|
||||
updated = "now"
|
||||
}
|
||||
out = append(out, strings.Join([]string{
|
||||
it.ExternalID, it.Title, it.ContentType, it.FileName, it.URL, updated, it.SourceResourceID,
|
||||
string(meta), string(it.Content),
|
||||
}, " | "))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sameItems(t *testing.T, label string, want, got []types.FetchedItem) {
|
||||
t.Helper()
|
||||
w, g := comparable(want), comparable(got)
|
||||
if strings.Join(w, "\n") != strings.Join(g, "\n") {
|
||||
t.Fatalf("%s differ\nbuiltin:\n%s\n\nplugin:\n%s", label, strings.Join(w, "\n"), strings.Join(g, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginMatchesBuiltinConnector(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
h, site := newHarness(t)
|
||||
|
||||
want, err := h.builtin.FetchAll(ctx, h.cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, pluginCursor := h.pluginFetch(pluginapi.FetchFull, nil)
|
||||
if len(want) != 3 {
|
||||
t.Fatalf("builtin fetched %d items, want 3", len(want))
|
||||
}
|
||||
sameItems(t, "full sync", want, got)
|
||||
if !strings.Contains(
|
||||
string(got[0].Content)+string(got[1].Content)+string(got[2].Content),
|
||||
"full text of the first article",
|
||||
) {
|
||||
t.Fatal("the article page should replace the feed teaser")
|
||||
}
|
||||
|
||||
// An incremental run with nothing changed emits nothing on either side.
|
||||
_, builtinCursor, err := h.builtin.FetchIncremental(ctx, h.cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unchanged, _, err := h.builtin.FetchIncremental(ctx, h.cfg, builtinCursor)
|
||||
if err != nil || len(unchanged) != 0 {
|
||||
t.Fatalf("builtin incremental without changes = %d items, %v", len(unchanged), err)
|
||||
}
|
||||
if again, _ := h.pluginFetch(pluginapi.FetchIncremental, pluginCursor); len(again) != 0 {
|
||||
t.Fatalf("plugin incremental without changes = %d items", len(again))
|
||||
}
|
||||
|
||||
// One entry changes: both emit exactly that entry.
|
||||
site.mu.Lock()
|
||||
site.secondA = "Second post, edited."
|
||||
site.mu.Unlock()
|
||||
changedWant, _, err := h.builtin.FetchIncremental(ctx, h.cfg, builtinCursor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changedGot, _ := h.pluginFetch(pluginapi.FetchIncremental, pluginCursor)
|
||||
if len(changedWant) != 1 {
|
||||
t.Fatalf("builtin incremental after an edit = %d items", len(changedWant))
|
||||
}
|
||||
sameItems(t, "incremental sync", changedWant, changedGot)
|
||||
}
|
||||
|
||||
func TestPluginValidatesLikeTheBuiltin(t *testing.T) {
|
||||
h, site := newHarness(t)
|
||||
ctx := context.Background()
|
||||
call := func(settings map[string]any) error {
|
||||
return h.plugin.Call(ctx, pluginapi.ConnectorValidatePath("rss"), pluginapi.Envelope{Config: pluginapi.Config{
|
||||
Instance: map[string]any{"settings": settings, "credentials": map[string]any{}},
|
||||
}}, nil, nil)
|
||||
}
|
||||
if err := call(h.cfg.Settings); err != nil {
|
||||
t.Fatalf("valid feeds rejected: %v", err)
|
||||
}
|
||||
for name, settings := range map[string]map[string]any{
|
||||
"no feeds": {},
|
||||
"missing feed": {"feed_urls": site.URL + "/nope.xml"},
|
||||
} {
|
||||
err := call(settings)
|
||||
e, ok := pluginapi.AsError(err)
|
||||
if !ok || e.Code != pluginapi.CodeInvalidConfig {
|
||||
t.Errorf("%s: want invalid_config, got %v", name, err)
|
||||
}
|
||||
cfg := &types.DataSourceConfig{Settings: settings}
|
||||
if h.builtin.Validate(ctx, cfg) == nil {
|
||||
t.Errorf("%s: the builtin accepts what the plugin rejects", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The example must stay an external plugin: the SDK and ordinary libraries,
|
||||
// nothing from WeKnora's internals.
|
||||
func TestPluginImportsNoInternals(t *testing.T) {
|
||||
files, _ := filepath.Glob("*.go")
|
||||
fset := token.NewFileSet()
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
src, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parsed, err := parser.ParseFile(fset, f, src, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, imp := range parsed.Imports {
|
||||
if strings.Contains(imp.Path.Value, "github.com/Tencent/WeKnora/internal") {
|
||||
t.Errorf("%s imports %s", f, imp.Path.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginPassesConformance(t *testing.T) {
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: "weknora-examples.rss", Version: Version})
|
||||
p.Connector("rss", &connector{})
|
||||
srv := httptest.NewServer(p.Handler())
|
||||
defer srv.Close()
|
||||
rep := conformance.Run(context.Background(), conformance.Target{
|
||||
Client: client.New(srv.URL, nil, nil),
|
||||
Raw: func(ctx context.Context, path string, body []byte) (*http.Response, error) {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+path, bytes.NewReader(body))
|
||||
return http.DefaultClient.Do(req)
|
||||
},
|
||||
})
|
||||
for _, r := range rep.Results {
|
||||
if !r.Passed {
|
||||
t.Errorf("%s: %s", r.Name, r.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
schemaVersion: 1
|
||||
id: weknora-examples.rss
|
||||
version: 1.0.0
|
||||
apiVersion: weknora.plugin/v1
|
||||
name: { en-US: RSS (plugin), zh-CN: RSS 订阅(插件版) }
|
||||
description:
|
||||
en-US: Sync RSS, Atom and JSON feeds into a knowledge base. The builtin RSS connector, rebuilt as a code plugin.
|
||||
zh-CN: 把 RSS、Atom、JSON Feed 同步到知识库。内置 RSS 连接器的插件实现。
|
||||
publisher: { id: weknora-examples, name: WeKnora examples }
|
||||
homepage: https://github.com/Tencent/WeKnora/tree/main/examples/plugins/rss
|
||||
license: MIT
|
||||
icon: icon.svg
|
||||
runtime: { type: host, kind: binary, entry: "bin/{os}-{arch}/rss" }
|
||||
permissions:
|
||||
# Feed items link to pages anywhere; private addresses stay unreachable.
|
||||
egress: ["*"]
|
||||
contributes:
|
||||
connectors:
|
||||
- id: rss
|
||||
name: { en-US: RSS (plugin), zh-CN: RSS 订阅(插件版) }
|
||||
description:
|
||||
en-US: Articles from RSS, Atom and JSON feeds, as full text
|
||||
zh-CN: 抓取 RSS、Atom、JSON Feed 的文章全文
|
||||
icon: icon.svg
|
||||
capabilities: [incremental, streaming]
|
||||
instanceSchema: schemas/instance.yaml
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
type: object
|
||||
required: [feed_urls]
|
||||
properties:
|
||||
feed_urls:
|
||||
type: string
|
||||
title: Feed URLs
|
||||
description: One feed per line
|
||||
x-widget: textarea
|
||||
x-group: settings
|
||||
x-placeholder: https://example.com/feed.xml
|
||||
x-i18n:
|
||||
title: { zh-CN: 订阅地址 }
|
||||
description: { zh-CN: 每行一个 }
|
||||
auth_headers:
|
||||
type: string
|
||||
title: Request headers
|
||||
description: "Optional, one per line as Name: Value; sent to feed URLs only"
|
||||
x-secret: true
|
||||
x-widget: textarea
|
||||
x-i18n:
|
||||
title: { zh-CN: 请求头 }
|
||||
description: { zh-CN: "可选,每行一个,格式为 名称: 值;只发给订阅地址" }
|
||||
|
|
@ -80,6 +80,13 @@ export interface ConnectorMeta {
|
|||
required_permissions?: string[]
|
||||
// Credential form: describes config.credentials for this connector.
|
||||
config_schema?: ConfigSchema
|
||||
// Settings form (config.settings) for connectors whose settings UI is
|
||||
// not built in, such as plugin connectors.
|
||||
settings_schema?: ConfigSchema
|
||||
// Localized name / description of plugin connectors, keyed by locale.
|
||||
names?: Record<string, string>
|
||||
descriptions?: Record<string, string>
|
||||
plugin_id?: string
|
||||
}
|
||||
|
||||
export interface Resource {
|
||||
|
|
@ -126,8 +133,12 @@ export function validateConnection(id: string) {
|
|||
}
|
||||
|
||||
// Validate credentials without persisting (during creation or credential replacement).
|
||||
export function validateCredentials(type: string, credentials: Record<string, any>) {
|
||||
return post('/api/v1/datasource/validate-credentials', { type, credentials })
|
||||
export function validateCredentials(
|
||||
type: string,
|
||||
credentials: Record<string, any>,
|
||||
settings?: Record<string, any>,
|
||||
) {
|
||||
return post('/api/v1/datasource/validate-credentials', { type, credentials, settings })
|
||||
}
|
||||
|
||||
// listResources lists selectable resources for a data source. Pass parentId to
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ export interface PluginVersion {
|
|||
/** This node's report on an installed plugin. */
|
||||
export interface PluginNodeStatus {
|
||||
version: string
|
||||
state: 'ready' | 'failed'
|
||||
// degraded: loaded, but its process crashed or fails health checks and is
|
||||
// being restarted.
|
||||
state: 'ready' | 'failed' | 'degraded'
|
||||
error?: string
|
||||
updatedAt: string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ export interface WebSearchProviderEntity {
|
|||
id?: string
|
||||
tenant_id?: number
|
||||
name: string
|
||||
provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply'
|
||||
// A builtin provider ID, or a plugin provider's qualified ID (acme.search/brave).
|
||||
provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply' | (string & {})
|
||||
description?: string
|
||||
parameters: {
|
||||
// api_key is never returned by the server in this shape; it lives behind
|
||||
|
|
@ -40,6 +41,10 @@ export interface WebSearchProviderTypeInfo {
|
|||
config_fields?: WebSearchProviderConfigField[]
|
||||
// The parameters form as a config schema; the settings drawer renders it.
|
||||
config_schema?: ConfigSchema
|
||||
// Plugin provider types: localized names, an icon (data: URI), the plugin.
|
||||
names?: Record<string, string>
|
||||
icon?: string
|
||||
plugin_id?: string
|
||||
}
|
||||
|
||||
export interface WebSearchProviderConfigField {
|
||||
|
|
|
|||
|
|
@ -116,6 +116,11 @@ const EXTRA_PREFIXES = [
|
|||
// knowledgeEditor.wikiBrowser.gallery.attr.builtin_caption …, and
|
||||
// everything else falls back to the contract wording.
|
||||
'knowledgeEditor.wikiBrowser.gallery.attr.',
|
||||
// Connector names and descriptions are keyed by the connector type the
|
||||
// backend registry lists (views/knowledge/settings/connectorLabels.ts);
|
||||
// plugin connectors fall back to their own names.
|
||||
'datasource.connector.',
|
||||
'datasource.connectorDesc.',
|
||||
] as const
|
||||
|
||||
/** Keys that must survive pruning even when static analysis misses them. */
|
||||
|
|
|
|||
|
|
@ -812,6 +812,8 @@ export default {
|
|||
channelGitLab: 'GitLab',
|
||||
channelIma: 'Tencent IMA',
|
||||
channelUpload: 'Upload',
|
||||
channelRss: 'RSS',
|
||||
channelExternal: 'External source',
|
||||
channelManual: 'Manual',
|
||||
channelUrl: 'Web',
|
||||
channelUnknown: 'Unknown',
|
||||
|
|
@ -2732,6 +2734,10 @@ export default {
|
|||
}
|
||||
},
|
||||
pluginAdmin: {
|
||||
runtime: {
|
||||
declarative: 'Declarative',
|
||||
host: 'Local process'
|
||||
},
|
||||
title: 'Plugin management',
|
||||
description: 'Install and manage plugins for the whole platform. Every workspace sees an installed plugin, and each one enables it for itself; disabling a plugin here unloads it on every node.',
|
||||
installButton: 'Install plugin',
|
||||
|
|
@ -2745,6 +2751,7 @@ export default {
|
|||
contributions: 'What it adds',
|
||||
permissions: 'Permissions and remote access',
|
||||
state: {
|
||||
degraded: 'Restarting',
|
||||
running: 'Running',
|
||||
failed: 'Failed to load',
|
||||
disabled: 'Disabled',
|
||||
|
|
@ -2767,6 +2774,7 @@ export default {
|
|||
reinstall: 'Reinstall'
|
||||
},
|
||||
permission: {
|
||||
anyHost: 'Any public host',
|
||||
remote: 'Remote service',
|
||||
egress: 'Network egress',
|
||||
hostApi: 'WeKnora API',
|
||||
|
|
|
|||
|
|
@ -812,6 +812,8 @@ export default {
|
|||
channelGitLab: 'GitLab',
|
||||
channelIma: 'Tencent IMA',
|
||||
channelUpload: 'アップロード',
|
||||
channelRss: 'RSS',
|
||||
channelExternal: '外部データソース',
|
||||
channelManual: '手動作成',
|
||||
channelUrl: 'Web',
|
||||
channelUnknown: '不明',
|
||||
|
|
@ -2732,6 +2734,10 @@ export default {
|
|||
}
|
||||
},
|
||||
pluginAdmin: {
|
||||
runtime: {
|
||||
declarative: '宣言型',
|
||||
host: 'ローカルプロセス'
|
||||
},
|
||||
title: 'プラグイン管理',
|
||||
description: 'プラットフォーム全体のプラグインをインストール・管理します。インストールしたプラグインはすべてのワークスペースに表示され、各ワークスペースで個別に有効化します。ここで無効にすると全ノードでアンロードされます。',
|
||||
installButton: 'プラグインをインストール',
|
||||
|
|
@ -2745,6 +2751,7 @@ export default {
|
|||
contributions: '提供する機能',
|
||||
permissions: '権限と外部アクセス',
|
||||
state: {
|
||||
degraded: '再起動中',
|
||||
running: '稼働中',
|
||||
failed: '読み込み失敗',
|
||||
disabled: '無効',
|
||||
|
|
@ -2767,6 +2774,7 @@ export default {
|
|||
reinstall: '再インストール'
|
||||
},
|
||||
permission: {
|
||||
anyHost: '任意の公開ホスト',
|
||||
remote: 'リモートサービス',
|
||||
egress: 'ネットワーク送信',
|
||||
hostApi: 'WeKnora API',
|
||||
|
|
|
|||
|
|
@ -5252,6 +5252,10 @@ export default {
|
|||
noCompatibleKbForAgent: '현재 에이전트의 도구와 범위 내 지식베이스의 기능이 일치하지 않아 참조할 수 있는 지식베이스가 없습니다.'
|
||||
},
|
||||
pluginAdmin: {
|
||||
runtime: {
|
||||
declarative: '선언형',
|
||||
host: '로컬 프로세스'
|
||||
},
|
||||
title: '플러그인 관리',
|
||||
description: '플랫폼 전체의 플러그인을 설치하고 관리합니다. 설치된 플러그인은 모든 워크스페이스에 보이며 각 워크스페이스가 직접 활성화합니다. 여기서 비활성화하면 모든 노드에서 언로드됩니다.',
|
||||
installButton: '플러그인 설치',
|
||||
|
|
@ -5265,6 +5269,7 @@ export default {
|
|||
contributions: '제공 기능',
|
||||
permissions: '권한 및 외부 접근',
|
||||
state: {
|
||||
degraded: '재시작 중',
|
||||
running: '실행 중',
|
||||
failed: '로드 실패',
|
||||
disabled: '비활성',
|
||||
|
|
@ -5287,6 +5292,7 @@ export default {
|
|||
reinstall: '재설치'
|
||||
},
|
||||
permission: {
|
||||
anyHost: '모든 공개 호스트',
|
||||
remote: '원격 서비스',
|
||||
egress: '네트워크 송신',
|
||||
hostApi: 'WeKnora API',
|
||||
|
|
@ -7315,6 +7321,8 @@ export default {
|
|||
channelGitLab: 'GitLab',
|
||||
channelIma: 'Tencent IMA',
|
||||
channelUpload: '업로드',
|
||||
channelRss: 'RSS',
|
||||
channelExternal: '외부 데이터 소스',
|
||||
channelManual: '수동',
|
||||
channelUrl: '웹',
|
||||
channelUnknown: '알 수 없음',
|
||||
|
|
|
|||
|
|
@ -5252,6 +5252,10 @@ export default {
|
|||
noCompatibleKbForAgent: 'Инструменты текущего агента не соответствуют возможностям ни одной базы знаний в области видимости — нечего упомянуть.'
|
||||
},
|
||||
pluginAdmin: {
|
||||
runtime: {
|
||||
declarative: 'Декларативный',
|
||||
host: 'Локальный процесс'
|
||||
},
|
||||
title: 'Управление плагинами',
|
||||
description: 'Установка и управление плагинами всей платформы. Установленный плагин виден всем рабочим пространствам, и каждое включает его самостоятельно; отключение здесь выгружает плагин на всех узлах.',
|
||||
installButton: 'Установить плагин',
|
||||
|
|
@ -5265,6 +5269,7 @@ export default {
|
|||
contributions: 'Что добавляет',
|
||||
permissions: 'Разрешения и внешний доступ',
|
||||
state: {
|
||||
degraded: 'Перезапуск',
|
||||
running: 'Работает',
|
||||
failed: 'Ошибка загрузки',
|
||||
disabled: 'Отключён',
|
||||
|
|
@ -5287,6 +5292,7 @@ export default {
|
|||
reinstall: 'Переустановка'
|
||||
},
|
||||
permission: {
|
||||
anyHost: 'Любой публичный хост',
|
||||
remote: 'Удалённый сервис',
|
||||
egress: 'Исходящая сеть',
|
||||
hostApi: 'API WeKnora',
|
||||
|
|
@ -7315,6 +7321,8 @@ export default {
|
|||
channelGitLab: 'GitLab',
|
||||
channelIma: 'Tencent IMA',
|
||||
channelUpload: 'Загрузка',
|
||||
channelRss: 'RSS',
|
||||
channelExternal: 'Внешний источник',
|
||||
channelManual: 'Вручную',
|
||||
channelUrl: 'Веб',
|
||||
channelUnknown: 'Неизвестно',
|
||||
|
|
|
|||
|
|
@ -5254,6 +5254,10 @@ export default {
|
|||
noCompatibleKbForAgent: '当前智能体的工具与作用域内知识库的能力不匹配,暂无可引用的知识库。'
|
||||
},
|
||||
pluginAdmin: {
|
||||
runtime: {
|
||||
declarative: '声明式',
|
||||
host: '本机进程'
|
||||
},
|
||||
title: '插件管理',
|
||||
description: '安装与管理本平台的插件。安装后所有空间都能看到,但每个空间需自行启用;停用会在所有节点卸载该插件。',
|
||||
installButton: '安装插件',
|
||||
|
|
@ -5267,6 +5271,7 @@ export default {
|
|||
contributions: '提供的能力',
|
||||
permissions: '权限与外部访问',
|
||||
state: {
|
||||
degraded: '重启中',
|
||||
running: '运行中',
|
||||
failed: '加载失败',
|
||||
disabled: '已停用',
|
||||
|
|
@ -5289,6 +5294,7 @@ export default {
|
|||
reinstall: '重新安装'
|
||||
},
|
||||
permission: {
|
||||
anyHost: '任意公网地址',
|
||||
remote: '访问远程服务',
|
||||
egress: '网络出口',
|
||||
hostApi: '调用 WeKnora API',
|
||||
|
|
@ -7317,6 +7323,8 @@ export default {
|
|||
channelGitLab: 'GitLab',
|
||||
channelIma: '腾讯 IMA',
|
||||
channelUpload: '上传',
|
||||
channelRss: 'RSS 订阅',
|
||||
channelExternal: '外部数据源',
|
||||
channelManual: '手动',
|
||||
channelUrl: '网页',
|
||||
channelUnknown: '未知',
|
||||
|
|
|
|||
|
|
@ -9,3 +9,18 @@ export function localizedText(text: LocalizedText | undefined, locale: string):
|
|||
const match = Object.keys(text).sort().find((k) => k !== 'default' && k.split('-')[0] === lang && text[k])
|
||||
return match ? text[match] : text.default
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks a variant from a locale map without a default ({ 'zh-CN': ... }, as
|
||||
* plugin connectors and provider types send names): exact, then same
|
||||
* language, else undefined.
|
||||
*/
|
||||
export function pickLocale(variants: Record<string, string> | undefined, locale: string): string | undefined {
|
||||
if (!variants) return undefined
|
||||
if (variants[locale]) return variants[locale]
|
||||
const lang = locale.split('-')[0].toLowerCase()
|
||||
const key = Object.keys(variants)
|
||||
.sort()
|
||||
.find((k) => k.split('-')[0].toLowerCase() === lang)
|
||||
return key ? variants[key] : undefined
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,6 +109,9 @@ const getSourceInfo = (item: KnowledgeItem): { icon: string; label: string } =>
|
|||
if (ch === 'dingtalk') return { icon: 'cloud-download', label: t('knowledgeBase.channelDingtalk') };
|
||||
if (ch === 'slack') return { icon: 'cloud-download', label: t('knowledgeBase.channelSlack') };
|
||||
if (ch === 'im') return { icon: 'cloud-download', label: t('knowledgeBase.channelIm') };
|
||||
if (ch === 'rss') return { icon: 'cloud-download', label: t('knowledgeBase.channelRss') };
|
||||
// Plugin connectors ingest under their qualified type ID (acme.feeds/feed).
|
||||
if (ch && ch.includes('/')) return { icon: 'cloud-download', label: t('knowledgeBase.channelExternal') };
|
||||
if (item.type === 'url') return { icon: 'link', label: t('knowledgeBase.channelUrl') };
|
||||
if (item.type === 'manual') return { icon: 'edit', label: t('knowledgeBase.channelManual') };
|
||||
return { icon: 'upload', label: t('knowledgeBase.channelUpload') };
|
||||
|
|
|
|||
|
|
@ -80,7 +80,15 @@ async function fixture({
|
|||
exports,
|
||||
require(name: string) {
|
||||
if (name === 'vue') return require('vue')
|
||||
if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key }) }
|
||||
if (name === 'vue-i18n') {
|
||||
return { useI18n: () => ({ t: (key: string) => key, te: () => true, locale: { value: 'en-US' } }) }
|
||||
}
|
||||
if (name === './connectorLabels') {
|
||||
return {
|
||||
connectorName: (d: { type: string }) => `datasource.connector.${d.type}`,
|
||||
connectorDescription: (d: { type: string }) => `datasource.connectorDesc.${d.type}`,
|
||||
}
|
||||
}
|
||||
if (name === 'tdesign-vue-next') return { MessagePlugin: { warning() {}, success() {}, error() {} } }
|
||||
if (name === '@/api/datasource') return api
|
||||
if (name === '@/components/schema-form/schema') return schemaModule
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
import { useSchemaText } from '@/components/schema-form/useSchemaText'
|
||||
import SettingDrawer from '@/components/settings/SettingDrawer.vue'
|
||||
import DataSourceTypeIcon from './DataSourceTypeIcon.vue'
|
||||
import { connectorDescription, connectorName } from './connectorLabels'
|
||||
import { getDatasourceIconUrl } from './datasourceIcons'
|
||||
|
||||
const props = defineProps<{
|
||||
|
|
@ -38,7 +39,7 @@ const props = defineProps<{
|
|||
|
||||
const visible = defineModel<boolean>('visible', { default: false })
|
||||
const emit = defineEmits<{ saved: [] }>()
|
||||
const { t } = useI18n()
|
||||
const { t, te, locale } = useI18n()
|
||||
|
||||
const isEdit = computed(() => !!props.dataSource)
|
||||
const step = ref(0)
|
||||
|
|
@ -536,6 +537,11 @@ const currentDef = computed(() => connectorTypes.value.find(d => d.type === form
|
|||
|
||||
const EMPTY_SCHEMA: ConfigSchema = { type: 'object', properties: {} }
|
||||
const credentialSchema = computed<ConfigSchema>(() => currentDef.value?.config_schema ?? EMPTY_SCHEMA)
|
||||
// Settings form of connectors without a built-in settings UI (plugins).
|
||||
const settingsSchema = computed<ConfigSchema | null>(() => currentDef.value?.settings_schema ?? null)
|
||||
const settingsErrors = ref<FieldError[]>([])
|
||||
const labelI18n = computed(() => ({ t: (k: string) => t(k), te: (k: string) => te(k), locale: locale.value }))
|
||||
const labelOf = (def: ConnectorMeta) => connectorName(def, labelI18n.value)
|
||||
|
||||
// Field errors of the last credential check, shown under each field.
|
||||
const credentialErrors = ref<FieldError[]>([])
|
||||
|
|
@ -547,9 +553,12 @@ watch(() => form.value.config.credentials, () => { credentialErrors.value = [] }
|
|||
// fields and warns with the first one, as the editor always has.
|
||||
function checkCredentials(): boolean {
|
||||
credentialErrors.value = validateConfig(credentialSchema.value, form.value.config.credentials)
|
||||
const first = credentialErrors.value[0]
|
||||
settingsErrors.value = settingsSchema.value ? validateConfig(settingsSchema.value, form.value.config.settings) : []
|
||||
const inCredentials = credentialErrors.value.length > 0
|
||||
const first = credentialErrors.value[0] ?? settingsErrors.value[0]
|
||||
if (!first) return true
|
||||
const field = schemaAt(credentialSchema.value, first.path)
|
||||
const schema = inCredentials ? credentialSchema.value : (settingsSchema.value as ConfigSchema)
|
||||
const field = schemaAt(schema, first.path)
|
||||
const label = field ? schemaText(field, 'title') : first.path
|
||||
MessagePlugin.warning(
|
||||
first.code === 'required'
|
||||
|
|
@ -673,9 +682,13 @@ watch(
|
|||
|
||||
function selectType(def: ConnectorMeta) {
|
||||
form.value.type = def.type
|
||||
form.value.name = t(`datasource.connector.${def.type}`)
|
||||
form.value.name = labelOf(def)
|
||||
// Schema defaults, e.g. Confluence starts on the Server edition.
|
||||
form.value.config.credentials = applyDefaults(def.config_schema ?? EMPTY_SCHEMA, {})
|
||||
if (def.settings_schema) {
|
||||
form.value.config.settings = applyDefaults(def.settings_schema, form.value.config.settings ?? {})
|
||||
}
|
||||
settingsErrors.value = []
|
||||
credentialErrors.value = []
|
||||
if (def.type === 'confluence') {
|
||||
form.value.config.settings = { ...form.value.config.settings, edition: 'server' }
|
||||
|
|
@ -717,7 +730,7 @@ async function testConnection() {
|
|||
// validate-credentials is credentials-only; feed URLs live in settings.
|
||||
creds.feed_urls = form.value.config.settings.feed_urls
|
||||
}
|
||||
await validateCredentials(form.value.type, creds)
|
||||
await validateCredentials(form.value.type, creds, form.value.config.settings)
|
||||
}
|
||||
testResult.value = 'success'
|
||||
MessagePlugin.success(t('datasource.testSuccess'))
|
||||
|
|
@ -1137,9 +1150,9 @@ const drawerConfirmText = computed(() => {
|
|||
@confirm="handleDrawerConfirm"
|
||||
@cancel="handleClose"
|
||||
>
|
||||
<template v-if="form.type && getDatasourceIconUrl(form.type)" #headerIcon>
|
||||
<template v-if="form.type && (getDatasourceIconUrl(form.type) || currentDef?.icon)" #headerIcon>
|
||||
<img
|
||||
:src="getDatasourceIconUrl(form.type)"
|
||||
:src="getDatasourceIconUrl(form.type) || currentDef?.icon"
|
||||
:alt="form.type"
|
||||
class="datasource-header-icon__img"
|
||||
>
|
||||
|
|
@ -1210,10 +1223,10 @@ const drawerConfirmText = computed(() => {
|
|||
@click="selectType(def)"
|
||||
>
|
||||
<div class="ds-type-header">
|
||||
<DataSourceTypeIcon :type="def.type" :size="20" />
|
||||
<span class="ds-type-name">{{ t(`datasource.connector.${def.type}`) }}</span>
|
||||
<DataSourceTypeIcon :type="def.type" :icon-url="def.icon" :label="labelOf(def)" :size="20" />
|
||||
<span class="ds-type-name">{{ labelOf(def) }}</span>
|
||||
</div>
|
||||
<div class="ds-type-desc">{{ t(`datasource.connectorDesc.${def.type}`) }}</div>
|
||||
<div class="ds-type-desc">{{ connectorDescription(def, labelI18n) }}</div>
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -1388,6 +1401,12 @@ const drawerConfirmText = computed(() => {
|
|||
:schema="credentialSchema"
|
||||
:errors="credentialErrors"
|
||||
/>
|
||||
<SchemaForm
|
||||
v-if="settingsSchema"
|
||||
v-model="form.config.settings"
|
||||
:schema="settingsSchema"
|
||||
:errors="settingsErrors"
|
||||
/>
|
||||
<div v-if="isEdit && replaceCredentialsMode" class="credential-edit-actions">
|
||||
<t-button size="small" variant="text" @click="cancelReplaceCredentials">
|
||||
{{ t('common.cancel') }}
|
||||
|
|
|
|||
|
|
@ -3,22 +3,36 @@ import { ref, computed, onMounted, onBeforeUnmount } from 'vue'
|
|||
import { MessagePlugin } from 'tdesign-vue-next'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import {
|
||||
getConnectorTypes,
|
||||
listDataSources,
|
||||
deleteDataSource,
|
||||
triggerSync,
|
||||
pauseDataSource,
|
||||
resumeDataSource,
|
||||
type ConnectorMeta,
|
||||
type DataSource,
|
||||
} from '@/api/datasource'
|
||||
import { humanizeCron, relativeTime } from '@/utils/cronHumanize'
|
||||
import DataSourceEditorDialog from './DataSourceEditorDialog.vue'
|
||||
import DataSourceSyncLogs from './DataSourceSyncLogs.vue'
|
||||
import DataSourceTypeIcon from './DataSourceTypeIcon.vue'
|
||||
import { connectorName } from './connectorLabels'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
|
||||
const props = defineProps<{ kbId: string }>()
|
||||
const emit = defineEmits<{ (e: 'count', value: number): void }>()
|
||||
const { t } = useI18n()
|
||||
const { t, te, locale } = useI18n()
|
||||
|
||||
// Connector metadata names plugin connectors, which have no locale keys.
|
||||
const connectorMeta = ref<Record<string, ConnectorMeta>>({})
|
||||
async function loadConnectorMeta() {
|
||||
try {
|
||||
const list = await getConnectorTypes()
|
||||
connectorMeta.value = Object.fromEntries([...list].map((m) => [m.type, m]))
|
||||
} catch {
|
||||
connectorMeta.value = {}
|
||||
}
|
||||
}
|
||||
const authStore = useAuthStore()
|
||||
|
||||
// 后端 /datasource 的 list/logs 是 Viewer+,但所有写操作(POST/PUT/DELETE
|
||||
|
|
@ -134,7 +148,7 @@ function syncModeLabel(mode: string) {
|
|||
}
|
||||
|
||||
function connectorLabel(type: string) {
|
||||
return t(`datasource.connector.${type}`) || type
|
||||
return connectorName(connectorMeta.value[type] ?? { type }, { t, te, locale: locale.value })
|
||||
}
|
||||
|
||||
function scheduleLabel(cron: string) {
|
||||
|
|
@ -177,7 +191,10 @@ function onEditorSaved() {
|
|||
loadList()
|
||||
}
|
||||
|
||||
onMounted(loadList)
|
||||
onMounted(() => {
|
||||
void loadConnectorMeta()
|
||||
void loadList()
|
||||
})
|
||||
onBeforeUnmount(stopPolling)
|
||||
</script>
|
||||
|
||||
|
|
@ -206,7 +223,12 @@ onBeforeUnmount(stopPolling)
|
|||
@click="canManageDataSource ? openEdit(ds) : undefined"
|
||||
>
|
||||
<div class="ds-card__badge">
|
||||
<DataSourceTypeIcon :type="ds.type" variant="badge" />
|
||||
<DataSourceTypeIcon
|
||||
:type="ds.type"
|
||||
:icon-url="connectorMeta[ds.type]?.icon"
|
||||
:label="connectorLabel(ds.type)"
|
||||
variant="badge"
|
||||
/>
|
||||
</div>
|
||||
<div class="ds-card__body">
|
||||
<div class="ds-card__header">
|
||||
|
|
|
|||
|
|
@ -3,10 +3,16 @@ import { getDatasourceIconUrl, datasourceIconMap } from './datasourceIcons'
|
|||
|
||||
const props = withDefaults(defineProps<{
|
||||
type: string
|
||||
/** Icon of a plugin connector (a data: URI from its metadata). */
|
||||
iconUrl?: string
|
||||
/** Display name, whose first letter stands in when there is no icon. */
|
||||
label?: string
|
||||
size?: number
|
||||
/** inline: 类型选择等小尺寸场景;badge: 嵌入 ds-card__badge 等父级徽章容器 */
|
||||
variant?: 'inline' | 'badge'
|
||||
}>(), {
|
||||
iconUrl: '',
|
||||
label: '',
|
||||
size: 20,
|
||||
variant: 'inline',
|
||||
})
|
||||
|
|
@ -26,7 +32,7 @@ function fallbackText(type: string) {
|
|||
case 'ima':
|
||||
return 'I'
|
||||
default:
|
||||
return type.slice(0, 1).toUpperCase() || '?'
|
||||
return (props.label || type).trim().slice(0, 1).toUpperCase() || '?'
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
@ -44,6 +50,13 @@ function fallbackText(type: string) {
|
|||
class="ds-type-icon__img"
|
||||
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
|
||||
>
|
||||
<img
|
||||
v-else-if="iconUrl"
|
||||
:src="iconUrl"
|
||||
:alt="label || type"
|
||||
class="ds-type-icon__img"
|
||||
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
|
||||
>
|
||||
<span v-else class="ds-type-icon-fallback">{{ fallbackText(type) }}</span>
|
||||
</span>
|
||||
</template>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
|
||||
import { connectorDescription, connectorName, isPluginChannel, pickLocale } from './connectorLabels'
|
||||
|
||||
const keys: Record<string, string> = { 'datasource.connector.rss': 'RSS', 'datasource.connectorDesc.rss': 'Feeds' }
|
||||
const i18n = (locale: string) => ({ t: (k: string) => keys[k] ?? k, te: (k: string) => k in keys, locale })
|
||||
|
||||
test('builtins use their locale keys', () => {
|
||||
assert.equal(connectorName({ type: 'rss', name: 'ignored' }, i18n('zh-CN')), 'RSS')
|
||||
assert.equal(connectorDescription({ type: 'rss' }, i18n('en-US')), 'Feeds')
|
||||
})
|
||||
|
||||
test('plugin connectors use their own localized names', () => {
|
||||
const plugin = {
|
||||
type: 'acme.feeds/feed',
|
||||
name: 'ACME Feed',
|
||||
names: { 'en-US': 'ACME Feed', 'zh-CN': 'ACME 订阅' },
|
||||
description: 'Feeds',
|
||||
}
|
||||
assert.equal(connectorName(plugin, i18n('zh-CN')), 'ACME 订阅')
|
||||
assert.equal(connectorName(plugin, i18n('zh-TW')), 'ACME 订阅')
|
||||
assert.equal(connectorName(plugin, i18n('ja-JP')), 'ACME Feed')
|
||||
assert.equal(connectorDescription(plugin, i18n('ja-JP')), 'Feeds')
|
||||
assert.equal(connectorName({ type: 'acme.x/y' }, i18n('en-US')), 'acme.x/y')
|
||||
})
|
||||
|
||||
test('pickLocale and plugin channels', () => {
|
||||
assert.equal(pickLocale({ 'zh-CN': 'a' }, 'zh-HK'), 'a')
|
||||
assert.equal(pickLocale(undefined, 'en-US'), undefined)
|
||||
assert.equal(isPluginChannel('acme.feeds/feed'), true)
|
||||
assert.equal(isPluginChannel('rss'), false)
|
||||
})
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
// Display names of data source connectors. Builtins have locale keys
|
||||
// (datasource.connector.<type>); plugin connectors bring localized names in
|
||||
// their metadata. Kept free of Vue so it runs under node:test.
|
||||
import { pickLocale } from '../../../utils/localizedText'
|
||||
|
||||
export { pickLocale }
|
||||
|
||||
/** The parts of connector metadata labels need. */
|
||||
export interface ConnectorLabelSource {
|
||||
type: string
|
||||
name?: string
|
||||
description?: string
|
||||
names?: Record<string, string>
|
||||
descriptions?: Record<string, string>
|
||||
}
|
||||
|
||||
/** The subset of vue-i18n the helpers use. */
|
||||
export interface LabelI18n {
|
||||
t: (key: string) => string
|
||||
te: (key: string) => boolean
|
||||
locale: string
|
||||
}
|
||||
|
||||
/** A connector's name: locale key, plugin variant, plugin default, type. */
|
||||
export function connectorName(src: ConnectorLabelSource, i18n: LabelI18n): string {
|
||||
const key = `datasource.connector.${src.type}`
|
||||
if (i18n.te(key)) return i18n.t(key)
|
||||
return pickLocale(src.names, i18n.locale) || src.name || src.type
|
||||
}
|
||||
|
||||
/** A connector's description, the same way; empty when there is none. */
|
||||
export function connectorDescription(src: ConnectorLabelSource, i18n: LabelI18n): string {
|
||||
const key = `datasource.connectorDesc.${src.type}`
|
||||
if (i18n.te(key)) return i18n.t(key)
|
||||
return pickLocale(src.descriptions, i18n.locale) || src.description || ''
|
||||
}
|
||||
|
||||
/** Channels plugin connectors ingest under: their qualified type ID. */
|
||||
export function isPluginChannel(channel: string | undefined): boolean {
|
||||
return !!channel && channel.includes('/')
|
||||
}
|
||||
|
|
@ -288,9 +288,10 @@ import CredentialResource, {
|
|||
import { useConfirmDelete } from '@/components/settings/useConfirmDelete'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
import { useChatResourcesStore } from '@/stores/chatResources'
|
||||
import { pickLocale } from '@/utils/localizedText'
|
||||
import { providerLogo } from './providerLogos'
|
||||
|
||||
const { t } = useI18n()
|
||||
const { t, locale } = useI18n()
|
||||
const authStore = useAuthStore()
|
||||
const chatResources = useChatResourcesStore()
|
||||
const confirmDelete = useConfirmDelete()
|
||||
|
|
@ -398,7 +399,7 @@ const drawerClass = computed(() => {
|
|||
// list card showed for the same provider id.
|
||||
const drawerLogo = computed(() => {
|
||||
const id = providerForm.value.provider
|
||||
return id ? providerLogo('websearch', id) : null
|
||||
return id ? resolveLogo(id) ?? null : null
|
||||
})
|
||||
|
||||
const drawerLogoStyle = computed((): Record<string, string> => {
|
||||
|
|
@ -424,7 +425,13 @@ const providerInitial = (providerId: string) => {
|
|||
}
|
||||
|
||||
// 见 VectorStoreSettings 的同名注释:返回 --logo-url 给 ::before 用 mask 渲染。
|
||||
const resolveLogo = (providerId: string) => providerLogo('websearch', providerId)
|
||||
// Plugin provider types carry their own icon.
|
||||
const resolveLogo = (providerId: string) => {
|
||||
const bundled = providerLogo('websearch', providerId)
|
||||
if (bundled) return bundled
|
||||
const icon = providerTypes.value.find((p) => p.id === providerId)?.icon
|
||||
return icon ? { mode: 'color' as const, url: icon } : undefined
|
||||
}
|
||||
|
||||
const badgeClass = (providerId: string) => {
|
||||
const m = resolveLogo(providerId)?.mode
|
||||
|
|
@ -470,7 +477,11 @@ const loadProviderEntities = async () => {
|
|||
|
||||
const loadProviderTypes = async () => {
|
||||
try {
|
||||
providerTypes.value = await listWebSearchProviderTypes()
|
||||
// Plugin provider types name themselves per locale.
|
||||
providerTypes.value = (await listWebSearchProviderTypes()).map((pt) => ({
|
||||
...pt,
|
||||
name: pickLocale(pt.names, locale.value) || pt.name,
|
||||
}))
|
||||
} catch (error) {
|
||||
console.error('Failed to load provider types:', error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@
|
|||
</t-tag>
|
||||
</t-tooltip>
|
||||
</div>
|
||||
<div class="plugin-card__meta">{{ p.id }} · v{{ p.active_version }} · {{ p.runtime }}</div>
|
||||
<div class="plugin-card__meta">{{ p.id }} · v{{ p.active_version }} · {{ runtimeLabel(p.runtime) }}</div>
|
||||
<div v-if="descriptionOf(p)" class="plugin-card__desc">{{ descriptionOf(p) }}</div>
|
||||
<div v-if="p.manifest" class="plugin-card__contribs">
|
||||
<span v-for="s in contributionSummary(p.manifest)" :key="s.point" class="plugin-card__contrib">
|
||||
|
|
@ -89,7 +89,8 @@ import { installedState } from './pluginManagementState'
|
|||
// Platform plugin management: what is installed, on which version, whether it
|
||||
// loads. Installing makes a plugin available to every workspace; each
|
||||
// workspace still turns it on in its own plugin center.
|
||||
const { t, locale } = useI18n()
|
||||
const { t, te, locale } = useI18n()
|
||||
const runtimeLabel = (rt: string) => (te(`pluginAdmin.runtime.${rt}`) ? t(`pluginAdmin.runtime.${rt}`) : rt)
|
||||
|
||||
const plugins = ref<InstalledPlugin[]>([])
|
||||
const loading = ref(false)
|
||||
|
|
@ -109,6 +110,8 @@ function stateTheme(p: InstalledPlugin) {
|
|||
return 'success'
|
||||
case 'failed':
|
||||
return 'danger'
|
||||
case 'degraded':
|
||||
return 'warning'
|
||||
default:
|
||||
return 'default'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ test('installedState reads desired state and the node report', () => {
|
|||
assert.equal(installedState(base), 'pending')
|
||||
assert.equal(installedState({ ...base, node: { version: '1', state: 'ready', updatedAt: '' } }), 'running')
|
||||
assert.equal(installedState({ ...base, node: { version: '1', state: 'failed', updatedAt: '' } }), 'failed')
|
||||
assert.equal(installedState({ ...base, node: { version: '1', state: 'degraded', updatedAt: '' } }), 'degraded')
|
||||
})
|
||||
|
||||
test('isPackageUrl and hasSystemConfig', () => {
|
||||
|
|
|
|||
|
|
@ -95,14 +95,24 @@ export function sortVersions(list: readonly PluginVersion[]): PluginVersion[] {
|
|||
}
|
||||
|
||||
/** Overall state of an installed plugin on the node that answered. */
|
||||
export type InstalledState = 'running' | 'failed' | 'disabled' | 'pending'
|
||||
export type InstalledState = 'running' | 'degraded' | 'failed' | 'disabled' | 'pending'
|
||||
|
||||
export function installedState(p: InstalledPlugin): InstalledState {
|
||||
if (p.desired_state === 'disabled') return 'disabled'
|
||||
if (!p.node) return 'pending'
|
||||
return p.node.state === 'ready' ? 'running' : 'failed'
|
||||
switch (p.node.state) {
|
||||
case 'ready':
|
||||
return 'running'
|
||||
case 'degraded':
|
||||
return 'degraded'
|
||||
default:
|
||||
return 'failed'
|
||||
}
|
||||
}
|
||||
|
||||
/** The egress grant meaning any public host. */
|
||||
export const EGRESS_ANY_HOST = '*'
|
||||
|
||||
/** Whether a string can be sent as a package URL. */
|
||||
export function isPackageUrl(raw: string): boolean {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
<dt>{{ t('pluginAdmin.publisher') }}</dt>
|
||||
<dd>{{ manifest?.publisher.name || manifest?.publisher.id }}</dd>
|
||||
<dt>{{ t('pluginAdmin.detail.runtime') }}</dt>
|
||||
<dd>{{ plugin.runtime }}</dd>
|
||||
<dd>{{ te(`pluginAdmin.runtime.${plugin.runtime}`) ? t(`pluginAdmin.runtime.${plugin.runtime}`) : plugin.runtime }}</dd>
|
||||
<dt>{{ t('pluginAdmin.detail.source') }}</dt>
|
||||
<dd>
|
||||
{{ t(`pluginAdmin.source.${plugin.source?.kind ?? 'upload'}`) }}
|
||||
|
|
@ -141,7 +141,7 @@ const emit = defineEmits<{
|
|||
removed: [id: string]
|
||||
}>()
|
||||
|
||||
const { t, locale } = useI18n()
|
||||
const { t, te, locale } = useI18n()
|
||||
|
||||
const manifest = computed(() => props.plugin?.manifest)
|
||||
const title = computed(() => (manifest.value ? localizedText(manifest.value.name, locale.value) : props.plugin?.id ?? ''))
|
||||
|
|
|
|||
|
|
@ -96,7 +96,10 @@
|
|||
</li>
|
||||
<li v-for="p in permissions" :key="`${p.kind}:${p.value}`">
|
||||
<span class="review-list__point">{{ t(`pluginAdmin.permission.${p.kind}`) }}</span>
|
||||
<code class="review-list__detail">{{ p.value }}</code>
|
||||
<strong v-if="p.kind === 'egress' && p.value === EGRESS_ANY_HOST" class="review-list__warn">
|
||||
{{ t('pluginAdmin.permission.anyHost') }}
|
||||
</strong>
|
||||
<code v-else class="review-list__detail">{{ p.value }}</code>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
|
@ -128,7 +131,15 @@ import {
|
|||
} from '@/api/system/plugins'
|
||||
import { localizedText } from '@/utils/localizedText'
|
||||
|
||||
import { hasSystemConfig, contributionLines, formatBytes, isPackageUrl, permissionLines, remoteHosts } from '../pluginManagementState'
|
||||
import {
|
||||
EGRESS_ANY_HOST,
|
||||
contributionLines,
|
||||
formatBytes,
|
||||
hasSystemConfig,
|
||||
isPackageUrl,
|
||||
permissionLines,
|
||||
remoteHosts,
|
||||
} from '../pluginManagementState'
|
||||
import { hasTenantConfig } from '../../settings/pluginCenterState'
|
||||
|
||||
// Installing is two steps on purpose: inspect shows what the package would
|
||||
|
|
@ -393,6 +404,12 @@ async function onConfirm() {
|
|||
color: var(--td-text-color-placeholder);
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
&__warn {
|
||||
font-size: var(--app-text-sm);
|
||||
font-weight: 500;
|
||||
color: var(--td-warning-color);
|
||||
}
|
||||
}
|
||||
|
||||
.review-alert {
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -100,6 +100,7 @@ require (
|
|||
github.com/JohannesKaufmann/dom v0.2.0 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/Tencent/WeKnora/pluginsdk v0.0.0-00010101000000-000000000000
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.1 // indirect
|
||||
|
|
@ -359,3 +360,5 @@ replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelg
|
|||
// The anydoc Go bindings are not published yet (firecrawl/anydoc#30 is still
|
||||
// open), so they are vendored. Drop this replace once upstream tags go/vX.Y.Z.
|
||||
replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go
|
||||
|
||||
replace github.com/Tencent/WeKnora/pluginsdk => ./pluginsdk
|
||||
|
|
|
|||
|
|
@ -658,7 +658,9 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
|||
}
|
||||
ds.ErrorMessage = syncLog.ErrorMessage
|
||||
_ = s.dsRepo.Update(ctx, ds)
|
||||
return err
|
||||
// Retrying cannot bring a connector back (its plugin was removed or
|
||||
// disabled); the next scheduled or manual sync will try again.
|
||||
return fmt.Errorf("%w: %w", asynq.SkipRetry, err)
|
||||
}
|
||||
|
||||
// Parse configuration
|
||||
|
|
@ -1247,7 +1249,9 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {
|
|||
}
|
||||
|
||||
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
|
||||
func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error {
|
||||
func (s *DataSourceService) ValidateCredentials(
|
||||
ctx context.Context, connectorType string, credentials, settings map[string]interface{},
|
||||
) error {
|
||||
connector, err := s.connectorRegistry.Get(connectorType)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -1255,6 +1259,7 @@ func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorTy
|
|||
config := &types.DataSourceConfig{
|
||||
Type: connectorType,
|
||||
Credentials: credentials,
|
||||
Settings: settings,
|
||||
}
|
||||
if err := connector.Validate(ctx, config); err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -14,11 +14,46 @@ import (
|
|||
// webSearchProviderService implements interfaces.WebSearchProviderService
|
||||
type webSearchProviderService struct {
|
||||
repo interfaces.WebSearchProviderRepository
|
||||
// registry knows the provider types installed plugins contribute.
|
||||
registry *infra_web_search.Registry
|
||||
}
|
||||
|
||||
// NewWebSearchProviderService creates a new web search provider service
|
||||
func NewWebSearchProviderService(repo interfaces.WebSearchProviderRepository) interfaces.WebSearchProviderService {
|
||||
return &webSearchProviderService{repo: repo}
|
||||
func NewWebSearchProviderService(
|
||||
repo interfaces.WebSearchProviderRepository, registry *infra_web_search.Registry,
|
||||
) interfaces.WebSearchProviderService {
|
||||
return &webSearchProviderService{repo: repo, registry: registry}
|
||||
}
|
||||
|
||||
// validType accepts builtin types and the types of loaded plugins.
|
||||
func (s *webSearchProviderService) validType(t types.WebSearchProviderType) bool {
|
||||
if isValidProviderType(t) {
|
||||
return true
|
||||
}
|
||||
if s.registry == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := s.registry.PluginType(string(t))
|
||||
return ok
|
||||
}
|
||||
|
||||
// validateParams checks builtin parameters by type and plugin parameters
|
||||
// against the plugin's instance schema.
|
||||
func (s *webSearchProviderService) validateParams(
|
||||
t types.WebSearchProviderType, params types.WebSearchProviderParameters,
|
||||
) error {
|
||||
if s.registry != nil {
|
||||
if pt, ok := s.registry.PluginType(string(t)); ok {
|
||||
if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
|
||||
return err
|
||||
}
|
||||
if pt.Validate != nil {
|
||||
return pt.Validate(params)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return validateProviderParameters(t, params)
|
||||
}
|
||||
|
||||
// CreateProvider creates a new web search provider configuration.
|
||||
|
|
@ -27,11 +62,11 @@ func (s *webSearchProviderService) CreateProvider(ctx context.Context, provider
|
|||
return fmt.Errorf("tenant ID is required")
|
||||
}
|
||||
|
||||
if !isValidProviderType(provider.Provider) {
|
||||
if !s.validType(provider.Provider) {
|
||||
return fmt.Errorf("invalid provider type: %s", provider.Provider)
|
||||
}
|
||||
|
||||
if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil {
|
||||
if err := s.validateParams(provider.Provider, provider.Parameters); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -52,7 +87,7 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
|
|||
}
|
||||
|
||||
// Validate provider type if set
|
||||
if provider.Provider != "" && !isValidProviderType(provider.Provider) {
|
||||
if provider.Provider != "" && !s.validType(provider.Provider) {
|
||||
return fmt.Errorf("invalid provider type: %s", provider.Provider)
|
||||
}
|
||||
|
||||
|
|
@ -63,7 +98,7 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
|
|||
}
|
||||
|
||||
if provider.Provider != "" {
|
||||
if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil {
|
||||
if err := s.validateParams(provider.Provider, provider.Parameters); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/plugin/activate"
|
||||
pluginbuiltin "github.com/Tencent/WeKnora/internal/plugin/builtin"
|
||||
plugindriver "github.com/Tencent/WeKnora/internal/plugin/driver"
|
||||
pluginhost "github.com/Tencent/WeKnora/internal/plugin/host"
|
||||
pluginmanifest "github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry"
|
||||
plugintenancy "github.com/Tencent/WeKnora/internal/plugin/tenancy"
|
||||
|
|
@ -165,6 +167,10 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
must(container.Provide(activate.NewMCPServers))
|
||||
must(container.Provide(activate.NewSkills))
|
||||
must(container.Provide(activate.NewModelVendors))
|
||||
must(container.Provide(pluginhost.NewManager))
|
||||
must(container.Provide(newPluginInvoker))
|
||||
must(container.Provide(activate.NewWebSearch))
|
||||
must(container.Provide(activate.NewConnectors))
|
||||
must(container.Provide(newMCPServiceRepository))
|
||||
must(container.Provide(repository.NewMCPToolApprovalRepository))
|
||||
must(container.Provide(repository.NewMCPOAuthRepository))
|
||||
|
|
@ -1835,7 +1841,8 @@ func newPluginRegistry(
|
|||
// declarative packages today; host, remote and kubernetes drivers join this
|
||||
// set.
|
||||
func newPluginDrivers(reg *pluginregistry.Registry, r *reconcile.Reconciler) *plugindriver.Set {
|
||||
return plugindriver.NewSet(plugindriver.NewBuiltin(reg.Plugin), r.Driver())
|
||||
return plugindriver.NewSet(plugindriver.NewBuiltin(reg.Plugin),
|
||||
r.Driver(pluginmanifest.RuntimeDeclarative), r.Driver(pluginmanifest.RuntimeHost))
|
||||
}
|
||||
|
||||
// installPluginGate gives the integration handlers the tenant plugin switches,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/handler"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/activate"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/host"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/install"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry"
|
||||
|
|
@ -34,13 +35,19 @@ func newPluginPackageStore(cfg *config.Config) (reconcile.PackageStore, error) {
|
|||
type pluginActivators struct {
|
||||
dig.In
|
||||
|
||||
MCP *activate.MCPServers
|
||||
Skills *activate.Skills
|
||||
Vendors *activate.ModelVendors
|
||||
Host *host.Manager
|
||||
WebSearch *activate.WebSearch
|
||||
Connectors *activate.Connectors
|
||||
MCP *activate.MCPServers
|
||||
Skills *activate.Skills
|
||||
Vendors *activate.ModelVendors
|
||||
Invoker *activate.Invoker
|
||||
}
|
||||
|
||||
// list orders the activators: the host first, so a code plugin's process is
|
||||
// running before anything routes calls to it.
|
||||
func (a pluginActivators) list() []reconcile.Activator {
|
||||
return []reconcile.Activator{a.Vendors, a.MCP, a.Skills}
|
||||
return []reconcile.Activator{a.Host, a.WebSearch, a.Connectors, a.Vendors, a.MCP, a.Skills}
|
||||
}
|
||||
|
||||
// bindPluginActivators hands the activators what they need once the plugin
|
||||
|
|
@ -50,10 +57,14 @@ func bindPluginActivators(
|
|||
skills *service.TenantSkillService,
|
||||
) {
|
||||
a.MCP.Bind(t, repo)
|
||||
a.Invoker.Bind(t, repo)
|
||||
a.Skills.Bind(t)
|
||||
skills.SetPluginSkills(a.Skills)
|
||||
}
|
||||
|
||||
// newPluginInvoker reaches code plugins through this node's plugin host.
|
||||
func newPluginInvoker(h *host.Manager) *activate.Invoker { return activate.NewInvoker(h) }
|
||||
|
||||
func newMCPServiceRepository(db *gorm.DB, plugins *activate.MCPServers) interfaces.MCPServiceRepository {
|
||||
return plugins.Repository(repository.NewMCPServiceRepository(db))
|
||||
}
|
||||
|
|
@ -80,8 +91,13 @@ func newPluginInstaller(
|
|||
|
||||
// startPluginReconciler loads installed plugins before the server takes
|
||||
// traffic, then keeps this node in step with the others.
|
||||
func startPluginReconciler(r *reconcile.Reconciler, cleaner interfaces.ResourceCleaner) {
|
||||
func startPluginReconciler(r *reconcile.Reconciler, hostManager *host.Manager, cleaner interfaces.ResourceCleaner) {
|
||||
hostManager.SetReporter(r)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r.Start(ctx)
|
||||
cleaner.RegisterWithName("PluginReconciler", func() error { cancel(); return nil })
|
||||
cleaner.RegisterWithName("PluginReconciler", func() error {
|
||||
cancel()
|
||||
hostManager.Close()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@ package datasource
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/plugin/configschema"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -121,15 +123,21 @@ type FullSyncWithCursor interface {
|
|||
) ([]types.FetchedItem, *types.SyncCursor, error)
|
||||
}
|
||||
|
||||
// ConnectorRegistry manages the registration and lookup of available connectors
|
||||
// ConnectorRegistry manages the registration and lookup of available
|
||||
// connectors. Installed plugins add and remove connectors while the server
|
||||
// runs, so it is safe for concurrent use.
|
||||
type ConnectorRegistry struct {
|
||||
mu sync.RWMutex
|
||||
connectors map[string]Connector
|
||||
// plugins holds the metadata of connectors installed plugins registered.
|
||||
plugins map[string]ConnectorMetadata
|
||||
}
|
||||
|
||||
// NewConnectorRegistry creates a new connector registry
|
||||
func NewConnectorRegistry() *ConnectorRegistry {
|
||||
return &ConnectorRegistry{
|
||||
connectors: make(map[string]Connector),
|
||||
plugins: make(map[string]ConnectorMetadata),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -141,12 +149,51 @@ func (r *ConnectorRegistry) Register(connector Connector) error {
|
|||
if connector.Type() == "" {
|
||||
return ErrConnectorTypeEmpty
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.connectors[connector.Type()] = connector
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterPlugin adds or replaces a plugin's connector with the metadata it
|
||||
// is listed under. It refuses to shadow a builtin connector.
|
||||
func (r *ConnectorRegistry) RegisterPlugin(connector Connector, meta ConnectorMetadata) error {
|
||||
if connector == nil {
|
||||
return ErrConnectorNil
|
||||
}
|
||||
t := connector.Type()
|
||||
if t == "" {
|
||||
return ErrConnectorTypeEmpty
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, exists := r.connectors[t]; exists {
|
||||
if _, isPlugin := r.plugins[t]; !isPlugin {
|
||||
return fmt.Errorf("connector type %s already exists", t)
|
||||
}
|
||||
}
|
||||
meta.Type = t
|
||||
r.connectors[t] = connector
|
||||
r.plugins[t] = meta
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unregister removes a plugin's connector; builtins stay. Syncs already
|
||||
// running keep the connector they started with.
|
||||
func (r *ConnectorRegistry) Unregister(connectorType string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, isPlugin := r.plugins[connectorType]; !isPlugin {
|
||||
return
|
||||
}
|
||||
delete(r.plugins, connectorType)
|
||||
delete(r.connectors, connectorType)
|
||||
}
|
||||
|
||||
// Get retrieves a connector by type
|
||||
func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
connector, exists := r.connectors[connectorType]
|
||||
if !exists {
|
||||
return nil, ErrConnectorNotFound
|
||||
|
|
@ -156,6 +203,8 @@ func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
|
|||
|
||||
// List returns all registered connector types
|
||||
func (r *ConnectorRegistry) List() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
types := make([]string, 0, len(r.connectors))
|
||||
for t := range r.connectors {
|
||||
types = append(types, t)
|
||||
|
|
@ -182,6 +231,16 @@ type ConnectorMetadata struct {
|
|||
// ConfigSchema describes DataSourceConfig.Credentials for this
|
||||
// connector; the editor renders the credential form from it.
|
||||
ConfigSchema *configschema.Schema `json:"config_schema,omitempty"`
|
||||
// SettingsSchema describes DataSourceConfig.Settings (non-secret
|
||||
// options) for connectors whose settings form is not built into the
|
||||
// frontend, such as plugin connectors.
|
||||
SettingsSchema *configschema.Schema `json:"settings_schema,omitempty"`
|
||||
// Names and Descriptions localize Name and Description, keyed by
|
||||
// locale, for plugin connectors (builtins use frontend locale keys).
|
||||
Names map[string]string `json:"names,omitempty"`
|
||||
Descriptions map[string]string `json:"descriptions,omitempty"`
|
||||
// PluginID names the installed plugin providing the connector.
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
}
|
||||
|
||||
// GetConnectorMetadata returns metadata for all available connectors
|
||||
|
|
@ -346,6 +405,8 @@ func ListAvailableConnectors() []ConnectorMetadata {
|
|||
// ListAvailableConnectors order. A registered connector without an entry in
|
||||
// ConnectorMetadataRegistry is listed last under its type.
|
||||
func (r *ConnectorRegistry) Metadata() []ConnectorMetadata {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
out := make([]ConnectorMetadata, 0, len(r.connectors))
|
||||
seen := make(map[string]bool, len(r.connectors))
|
||||
for _, meta := range ListAvailableConnectors() {
|
||||
|
|
@ -362,6 +423,10 @@ func (r *ConnectorRegistry) Metadata() []ConnectorMetadata {
|
|||
}
|
||||
sort.Strings(rest)
|
||||
for _, t := range rest {
|
||||
if meta, ok := r.plugins[t]; ok {
|
||||
out = append(out, meta)
|
||||
continue
|
||||
}
|
||||
out = append(out, ConnectorMetadata{Type: t, Name: t, Priority: math.MaxInt32})
|
||||
}
|
||||
return out
|
||||
|
|
|
|||
|
|
@ -326,13 +326,18 @@ func (h *DataSourceHandler) ValidateCredentials(c *gin.Context) {
|
|||
var req struct {
|
||||
Type string `json:"type" binding:"required"`
|
||||
Credentials map[string]interface{} `json:"credentials" binding:"required"`
|
||||
Settings map[string]interface{} `json:"settings"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: type and credentials are required"})
|
||||
return
|
||||
}
|
||||
if !h.pluginFilter(c)(manifest.PointConnectors, req.Type) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": disabledIntegrationError})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials); err != nil {
|
||||
if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials, req.Settings); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -329,6 +329,9 @@ func (h *WebSearchProviderHandler) DeleteProvider(c *gin.Context) {
|
|||
func (h *WebSearchProviderHandler) ListProviderTypes(c *gin.Context) {
|
||||
enabled := h.pluginFilter(c)
|
||||
all := types.GetWebSearchProviderTypes()
|
||||
if h.registry != nil {
|
||||
all = append(all, h.registry.PluginTypes()...)
|
||||
}
|
||||
out := make([]types.WebSearchProviderTypeInfo, 0, len(all))
|
||||
for _, info := range all {
|
||||
if enabled(manifest.PointWebSearch, info.ID) {
|
||||
|
|
@ -408,6 +411,11 @@ func (h *WebSearchProviderHandler) TestProviderRaw(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
if !h.pluginFilter(c)(manifest.PointWebSearch, req.Provider) {
|
||||
_ = c.Error(errors.NewBadRequestError(disabledIntegrationError))
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.doTestSearch(ctx, req.Provider, req.Parameters); err != nil {
|
||||
logger.Warnf(ctx, "Web search provider test failed: %v", err)
|
||||
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
|
||||
|
|
|
|||
|
|
@ -17,16 +17,74 @@ type ProviderFactory func(params types.WebSearchProviderParameters) (interfaces.
|
|||
// Instances are created on-demand with tenant-specific parameters.
|
||||
type Registry struct {
|
||||
factories map[string]ProviderFactory
|
||||
mu sync.RWMutex
|
||||
// plugins describes the types installed plugins registered; builtin
|
||||
// types are described by types.GetWebSearchProviderTypes.
|
||||
plugins map[string]PluginType
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// PluginType is a provider type an installed plugin contributes.
|
||||
type PluginType struct {
|
||||
Info types.WebSearchProviderTypeInfo
|
||||
// Validate checks instance parameters against the plugin's schema.
|
||||
Validate func(params types.WebSearchProviderParameters) error
|
||||
}
|
||||
|
||||
// NewRegistry creates a new web search provider registry
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{
|
||||
factories: make(map[string]ProviderFactory),
|
||||
plugins: make(map[string]PluginType),
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterPlugin adds or replaces a plugin's provider type. It refuses to
|
||||
// shadow a builtin type.
|
||||
func (r *Registry) RegisterPlugin(t PluginType, factory ProviderFactory) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
id := t.Info.ID
|
||||
if _, exists := r.factories[id]; exists {
|
||||
if _, isPlugin := r.plugins[id]; !isPlugin {
|
||||
return fmt.Errorf("web search provider type %s already exists", id)
|
||||
}
|
||||
}
|
||||
r.factories[id] = factory
|
||||
r.plugins[id] = t
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unregister removes a plugin's provider type; builtins stay.
|
||||
func (r *Registry) Unregister(id string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, isPlugin := r.plugins[id]; !isPlugin {
|
||||
return
|
||||
}
|
||||
delete(r.plugins, id)
|
||||
delete(r.factories, id)
|
||||
}
|
||||
|
||||
// PluginType returns a plugin provider type.
|
||||
func (r *Registry) PluginType(id string) (PluginType, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
t, ok := r.plugins[id]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// PluginTypes returns the plugin provider types, sorted by ID.
|
||||
func (r *Registry) PluginTypes() []types.WebSearchProviderTypeInfo {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
out := make([]types.WebSearchProviderTypeInfo, 0, len(r.plugins))
|
||||
for _, t := range r.plugins {
|
||||
out = append(out, t.Info)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out
|
||||
}
|
||||
|
||||
// Register registers a provider type factory by ID
|
||||
func (r *Registry) Register(id string, factory ProviderFactory) {
|
||||
r.mu.Lock()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,354 @@
|
|||
package activate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/configschema"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Connectors registers code plugins' data source connectors. A plugin
|
||||
// connector's instanceSchema splits in two: fields in x-group "settings"
|
||||
// are non-secret options (DataSourceConfig.Settings), everything else is
|
||||
// credentials (encrypted at rest).
|
||||
type Connectors struct {
|
||||
iv *Invoker
|
||||
registry *datasource.ConnectorRegistry
|
||||
|
||||
mu sync.Mutex
|
||||
registered map[string][]string
|
||||
}
|
||||
|
||||
// NewConnectors creates the connector activator.
|
||||
func NewConnectors(iv *Invoker, registry *datasource.ConnectorRegistry) *Connectors {
|
||||
return &Connectors{iv: iv, registry: registry, registered: map[string][]string{}}
|
||||
}
|
||||
|
||||
// Name implements reconcile.Activator.
|
||||
func (a *Connectors) Name() string { return "connectors" }
|
||||
|
||||
// settingsGroup marks instanceSchema fields that are settings, not
|
||||
// credentials.
|
||||
const settingsGroup = "settings"
|
||||
|
||||
// Activate implements reconcile.Activator: all of a plugin's connectors or
|
||||
// none.
|
||||
func (a *Connectors) Activate(_ context.Context, l *reconcile.Loaded) error {
|
||||
var ids []string
|
||||
for _, c := range l.Manifest.Contributes[manifest.PointConnectors] {
|
||||
id := manifest.QualifiedID(l.Manifest.ID, c.ID)
|
||||
meta, err := connectorMetadata(l, c, id)
|
||||
if err == nil {
|
||||
rc := &remoteConnector{iv: a.iv, m: l.Manifest, local: c.ID, typeID: id}
|
||||
err = a.registry.RegisterPlugin(rc, meta)
|
||||
}
|
||||
if err != nil {
|
||||
for _, done := range ids {
|
||||
a.registry.Unregister(done)
|
||||
}
|
||||
return fmt.Errorf("connector %s: %w", c.ID, err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.registered[l.Manifest.ID] = ids
|
||||
a.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deactivate implements reconcile.Activator.
|
||||
func (a *Connectors) Deactivate(_ context.Context, pluginID string) error {
|
||||
a.mu.Lock()
|
||||
ids := a.registered[pluginID]
|
||||
delete(a.registered, pluginID)
|
||||
a.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
a.registry.Unregister(id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func connectorMetadata(
|
||||
l *reconcile.Loaded,
|
||||
c manifest.Contribution,
|
||||
id string,
|
||||
) (datasource.ConnectorMetadata, error) {
|
||||
src, err := instanceSchema(l.Package, c)
|
||||
if err != nil {
|
||||
return datasource.ConnectorMetadata{}, err
|
||||
}
|
||||
creds, settings := splitConnectorSchema(src)
|
||||
icon, err := iconDataURI(l.Package, c.Icon)
|
||||
if err != nil {
|
||||
return datasource.ConnectorMetadata{}, err
|
||||
}
|
||||
caps := c.Capabilities
|
||||
if caps == nil {
|
||||
caps = []string{}
|
||||
}
|
||||
return datasource.ConnectorMetadata{
|
||||
Type: id, Name: c.Name.Default, Names: c.Name.Locales,
|
||||
Description: c.Description.Default, Descriptions: c.Description.Locales,
|
||||
Icon: icon, Priority: 1000 + c.Order, AuthType: "plugin", Capabilities: caps,
|
||||
ConfigSchema: creds, SettingsSchema: settings, PluginID: l.Manifest.ID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// splitConnectorSchema separates settings fields from credential fields.
|
||||
func splitConnectorSchema(src *configschema.Schema) (creds, settings *configschema.Schema) {
|
||||
if src == nil {
|
||||
return nil, nil
|
||||
}
|
||||
required := map[string]bool{}
|
||||
for _, k := range src.Required {
|
||||
required[k] = true
|
||||
}
|
||||
creds, settings = configschema.Object(), configschema.Object()
|
||||
keys := make([]string, 0, len(src.Properties))
|
||||
for k := range src.Properties {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
prop := *src.Properties[k]
|
||||
if prop.Group == settingsGroup && !prop.Secret {
|
||||
prop.Group = ""
|
||||
settings.Set(k, &prop, required[k])
|
||||
continue
|
||||
}
|
||||
creds.Set(k, &prop, required[k])
|
||||
}
|
||||
if len(creds.Properties) == 0 {
|
||||
creds = nil
|
||||
}
|
||||
if len(settings.Properties) == 0 {
|
||||
settings = nil
|
||||
}
|
||||
return creds, settings
|
||||
}
|
||||
|
||||
// remoteConnector is a plugin connector. It streams, so the sync service
|
||||
// interleaves fetch, ingest and checkpoint and resumes after a timeout.
|
||||
type remoteConnector struct {
|
||||
iv *Invoker
|
||||
m *manifest.Manifest
|
||||
local string
|
||||
typeID string
|
||||
}
|
||||
|
||||
var _ datasource.FullStreamingConnector = (*remoteConnector)(nil)
|
||||
|
||||
func (r *remoteConnector) Type() string { return r.typeID }
|
||||
|
||||
func instanceOf(cfg *types.DataSourceConfig) map[string]any {
|
||||
if cfg == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return map[string]any{
|
||||
"credentials": nonNil(cfg.Credentials),
|
||||
"settings": nonNil(cfg.Settings),
|
||||
"resourceIds": cfg.ResourceIDs,
|
||||
}
|
||||
}
|
||||
|
||||
func nonNil(m map[string]any) map[string]any {
|
||||
if m == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Validate implements datasource.Connector.
|
||||
func (r *remoteConnector) Validate(ctx context.Context, cfg *types.DataSourceConfig) error {
|
||||
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
|
||||
defer cancel()
|
||||
return r.iv.Call(ctx, r.m, pluginapi.ConnectorValidatePath(r.local), instanceOf(cfg), nil, nil)
|
||||
}
|
||||
|
||||
// ListResources implements datasource.Connector.
|
||||
func (r *remoteConnector) ListResources(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, parentID string,
|
||||
) ([]types.Resource, error) {
|
||||
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
|
||||
defer cancel()
|
||||
var out pluginapi.ListResourcesOutput
|
||||
err := r.iv.Call(ctx, r.m, pluginapi.ConnectorListResourcesPath(r.local), instanceOf(cfg),
|
||||
pluginapi.ListResourcesInput{ParentID: parentID}, &out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := make([]types.Resource, 0, len(out.Resources))
|
||||
for _, x := range out.Resources {
|
||||
item := types.Resource{
|
||||
ExternalID: x.ExternalID, Name: x.Name, Type: x.Type, Description: x.Description, URL: x.URL,
|
||||
ParentID: x.ParentID, HasChildren: x.HasChildren, Metadata: x.Metadata,
|
||||
}
|
||||
if x.ModifiedAt != nil {
|
||||
item.ModifiedAt = *x.ModifiedAt
|
||||
}
|
||||
res = append(res, item)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors implements datasource.Connector.
|
||||
func (r *remoteConnector) ResolveResourceAncestors(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
|
||||
defer cancel()
|
||||
var out pluginapi.ResolveAncestorsOutput
|
||||
err := r.iv.Call(ctx, r.m, pluginapi.ConnectorResolveAncestorsPath(r.local), instanceOf(cfg),
|
||||
pluginapi.ResolveAncestorsInput{ResourceIDs: resourceIDs}, &out)
|
||||
return out.Ancestors, err
|
||||
}
|
||||
|
||||
// collector gathers a stream for the non-streaming entry points.
|
||||
type collector struct {
|
||||
items []types.FetchedItem
|
||||
cursor *types.SyncCursor
|
||||
}
|
||||
|
||||
func (c *collector) Emit(_ context.Context, item types.FetchedItem) error {
|
||||
c.items = append(c.items, item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *collector) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
|
||||
c.cursor = cursor
|
||||
return nil
|
||||
}
|
||||
|
||||
// FetchAll implements datasource.Connector.
|
||||
func (r *remoteConnector) FetchAll(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]types.FetchedItem, error) {
|
||||
c := &collector{}
|
||||
scoped := *cfg
|
||||
if len(resourceIDs) > 0 {
|
||||
scoped.ResourceIDs = resourceIDs
|
||||
}
|
||||
_, err := r.fetch(ctx, &scoped, pluginapi.FetchFull, nil, c)
|
||||
return c.items, err
|
||||
}
|
||||
|
||||
// FetchIncremental implements datasource.Connector.
|
||||
func (r *remoteConnector) FetchIncremental(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor,
|
||||
) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
c := &collector{}
|
||||
next, err := r.fetch(ctx, cfg, pluginapi.FetchIncremental, cursor, c)
|
||||
return c.items, next, err
|
||||
}
|
||||
|
||||
// FetchStream implements datasource.StreamingConnector: incremental from a
|
||||
// cursor, full without one.
|
||||
func (r *remoteConnector) FetchStream(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
|
||||
) (*types.SyncCursor, error) {
|
||||
mode := pluginapi.FetchIncremental
|
||||
if cursor == nil {
|
||||
mode = pluginapi.FetchFull
|
||||
}
|
||||
return r.fetch(ctx, cfg, mode, cursor, h)
|
||||
}
|
||||
|
||||
// FetchFullStream implements datasource.FullStreamingConnector: every item
|
||||
// again, with the previous cursor for deletion reconciliation.
|
||||
func (r *remoteConnector) FetchFullStream(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
|
||||
) (*types.SyncCursor, error) {
|
||||
return r.fetch(ctx, cfg, pluginapi.FetchFull, cursor, h)
|
||||
}
|
||||
|
||||
func (r *remoteConnector) fetch(
|
||||
ctx context.Context, cfg *types.DataSourceConfig, mode string, cursor *types.SyncCursor,
|
||||
h datasource.StreamHandler,
|
||||
) (*types.SyncCursor, error) {
|
||||
in := pluginapi.FetchInput{Mode: mode, Cursor: toPluginCursor(cursor)}
|
||||
if cfg != nil {
|
||||
in.ResourceIDs = cfg.ResourceIDs
|
||||
}
|
||||
end, err := r.iv.Stream(ctx, r.m, pluginapi.ConnectorFetchPath(r.local), instanceOf(cfg), in,
|
||||
func(ev pluginapi.Event) error {
|
||||
switch ev.Type {
|
||||
case pluginapi.EventItem:
|
||||
var it pluginapi.FetchedItem
|
||||
if err := json.Unmarshal(ev.Data, &it); err != nil {
|
||||
return fmt.Errorf("decode fetched item: %w", err)
|
||||
}
|
||||
return h.Emit(ctx, fromPluginItem(it))
|
||||
case pluginapi.EventCheckpoint:
|
||||
var c pluginapi.Cursor
|
||||
if err := json.Unmarshal(ev.Data, &c); err != nil {
|
||||
return fmt.Errorf("decode checkpoint: %w", err)
|
||||
}
|
||||
return h.Checkpoint(ctx, fromPluginCursor(&c))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var final pluginapi.Cursor
|
||||
if len(end) > 0 {
|
||||
if err := json.Unmarshal(end, &final); err != nil {
|
||||
return nil, fmt.Errorf("decode final cursor: %w", err)
|
||||
}
|
||||
}
|
||||
next := fromPluginCursor(&final)
|
||||
if next.LastSyncTime.IsZero() {
|
||||
next.LastSyncTime = time.Now()
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func toPluginCursor(c *types.SyncCursor) *pluginapi.Cursor {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
out := &pluginapi.Cursor{State: c.ConnectorCursor}
|
||||
if !c.LastSyncTime.IsZero() {
|
||||
t := c.LastSyncTime
|
||||
out.LastSyncTime = &t
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fromPluginCursor(c *pluginapi.Cursor) *types.SyncCursor {
|
||||
out := &types.SyncCursor{ConnectorCursor: c.State}
|
||||
if c.LastSyncTime != nil {
|
||||
out.LastSyncTime = *c.LastSyncTime
|
||||
}
|
||||
if out.ConnectorCursor == nil {
|
||||
out.ConnectorCursor = map[string]any{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fromPluginItem(it pluginapi.FetchedItem) types.FetchedItem {
|
||||
out := types.FetchedItem{
|
||||
ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType,
|
||||
FileName: it.FileName, URL: it.URL, Metadata: it.Metadata, IsDeleted: it.IsDeleted,
|
||||
SourceResourceID: it.SourceResourceID,
|
||||
}
|
||||
if it.UpdatedAt != nil {
|
||||
out.UpdatedAt = *it.UpdatedAt
|
||||
}
|
||||
if it.CreatedAt != nil {
|
||||
out.CreatedAt = *it.CreatedAt
|
||||
}
|
||||
if out.ContentType == "" {
|
||||
out.ContentType = "text/markdown"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
package activate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/pkg"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/plugintest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
const feedPluginManifest = `schemaVersion: 1
|
||||
id: acme.feeds
|
||||
version: 1.0.0
|
||||
apiVersion: weknora.plugin/v1
|
||||
name: { en-US: ACME Feeds }
|
||||
publisher: { id: acme }
|
||||
runtime: { type: host, kind: binary, entry: bin/feeds }
|
||||
contributes:
|
||||
connectors:
|
||||
- id: feed
|
||||
name: { en-US: ACME Feed, zh-CN: ACME 订阅 }
|
||||
capabilities: [incremental]
|
||||
instanceSchema: schemas/feed.yaml
|
||||
`
|
||||
|
||||
const feedSchema = `type: object
|
||||
required: [feed_urls]
|
||||
properties:
|
||||
token: { type: string, x-secret: true }
|
||||
feed_urls: { type: string, x-widget: textarea, x-group: settings }
|
||||
`
|
||||
|
||||
// feedPlugin serves numbered items and resumes from the cursor.
|
||||
type feedPlugin struct{}
|
||||
|
||||
func (feedPlugin) Validate(_ context.Context, _ *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
|
||||
if cfg.Settings["feed_urls"] == nil {
|
||||
return pluginapi.InvalidConfig("feed_urls is required", map[string]string{"settings.feed_urls": "required"})
|
||||
}
|
||||
if cfg.Credentials["token"] != "t-1" {
|
||||
return pluginapi.Errorf(pluginapi.CodeUnauthorized, "bad token")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (feedPlugin) ListResources(
|
||||
_ context.Context,
|
||||
_ *pluginsdk.Call,
|
||||
cfg pluginsdk.ConnectorConfig,
|
||||
parent string,
|
||||
) ([]pluginapi.Resource, error) {
|
||||
if parent != "" {
|
||||
return nil, nil
|
||||
}
|
||||
var out []pluginapi.Resource
|
||||
for _, u := range strings.Split(fmt.Sprint(cfg.Settings["feed_urls"]), "\n") {
|
||||
out = append(out, pluginapi.Resource{ExternalID: u, Name: u, Type: "feed"})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (feedPlugin) Fetch(
|
||||
_ context.Context,
|
||||
_ *pluginsdk.Call,
|
||||
_ pluginsdk.ConnectorConfig,
|
||||
in pluginapi.FetchInput,
|
||||
s *pluginsdk.Stream,
|
||||
) (*pluginapi.Cursor, error) {
|
||||
start := 0
|
||||
if in.Mode == pluginapi.FetchIncremental && in.Cursor != nil {
|
||||
start = int(in.Cursor.State["seen"].(float64))
|
||||
}
|
||||
for i := start; i < 3; i++ {
|
||||
item := pluginapi.FetchedItem{
|
||||
ExternalID: fmt.Sprint("post-", i), Title: fmt.Sprint("Post ", i), Content: []byte("# hi"),
|
||||
}
|
||||
if err := s.Item(item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.Checkpoint(pluginapi.Cursor{State: map[string]any{"seen": i + 1}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &pluginapi.Cursor{State: map[string]any{"seen": 3}}, nil
|
||||
}
|
||||
|
||||
type recordingHandler struct {
|
||||
items []string
|
||||
checkpoints int
|
||||
}
|
||||
|
||||
func (h *recordingHandler) Emit(_ context.Context, item types.FetchedItem) error {
|
||||
h.items = append(h.items, item.ExternalID+"|"+item.ContentType)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *recordingHandler) Checkpoint(context.Context, *types.SyncCursor) error {
|
||||
h.checkpoints++
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPluginConnector(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
plugin := pluginsdk.New(pluginsdk.Info{ID: "acme.feeds", Version: "1.0.0"})
|
||||
plugin.Connector("feed", feedPlugin{})
|
||||
srv := httptest.NewServer(plugin.Handler())
|
||||
defer srv.Close()
|
||||
|
||||
p, err := pkg.Open(plugintest.Zip(t, map[string]string{
|
||||
"plugin.yaml": feedPluginManifest, "bin/feeds": "x", "schemas/feed.yaml": feedSchema,
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := datasource.NewConnectorRegistry()
|
||||
a := NewConnectors(NewInvoker(fakeClients{client.New(srv.URL, nil, nil)}), registry)
|
||||
if err := a.Activate(ctx, &reconcile.Loaded{Manifest: p.Manifest, Package: p}); err != nil {
|
||||
t.Fatalf("Activate: %v", err)
|
||||
}
|
||||
|
||||
var meta datasource.ConnectorMetadata
|
||||
for _, m := range registry.Metadata() {
|
||||
if m.Type == "acme.feeds/feed" {
|
||||
meta = m
|
||||
}
|
||||
}
|
||||
if meta.Names["zh-CN"] != "ACME 订阅" || meta.PluginID != "acme.feeds" ||
|
||||
meta.ConfigSchema.Properties["token"] == nil || meta.SettingsSchema.Properties["feed_urls"] == nil ||
|
||||
meta.ConfigSchema.Properties["feed_urls"] != nil {
|
||||
t.Fatalf("metadata = %+v", meta)
|
||||
}
|
||||
|
||||
conn, err := registry.Get("acme.feeds/feed")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &types.DataSourceConfig{
|
||||
Credentials: map[string]any{"token": "t-1"},
|
||||
Settings: map[string]any{"feed_urls": "https://a.example/rss\nhttps://b.example/rss"},
|
||||
}
|
||||
err = conn.Validate(ctx, &types.DataSourceConfig{Credentials: map[string]any{"token": "t-1"}})
|
||||
var pe *pluginapi.Error
|
||||
if err == nil || err.Error() != "feed_urls is required" ||
|
||||
!errors.As(err, &pe) || pe.Code != pluginapi.CodeInvalidConfig {
|
||||
t.Fatalf("Validate without settings = %v; want the plugin's message with its code reachable", err)
|
||||
}
|
||||
if err := conn.Validate(ctx, cfg); err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
}
|
||||
res, err := conn.ListResources(ctx, cfg, "")
|
||||
if err != nil || len(res) != 2 || res[1].ExternalID != "https://b.example/rss" {
|
||||
t.Fatalf("ListResources = %+v, %v", res, err)
|
||||
}
|
||||
|
||||
streaming := conn.(datasource.StreamingConnector)
|
||||
h := &recordingHandler{}
|
||||
cursor, err := streaming.FetchStream(ctx, cfg, nil, h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(h.items, ",") != "post-0|text/markdown,post-1|text/markdown,post-2|text/markdown" ||
|
||||
h.checkpoints != 3 || cursor.ConnectorCursor["seen"].(float64) != 3 || cursor.LastSyncTime.IsZero() {
|
||||
t.Fatalf("items=%v checkpoints=%d cursor=%+v", h.items, h.checkpoints, cursor)
|
||||
}
|
||||
cursor.ConnectorCursor["seen"] = float64(2)
|
||||
items, next, err := conn.FetchIncremental(ctx, cfg, cursor)
|
||||
if err != nil || len(items) != 1 || items[0].ExternalID != "post-2" ||
|
||||
next.ConnectorCursor["seen"].(float64) != 3 {
|
||||
t.Fatalf("incremental = %+v %+v %v", items, next, err)
|
||||
}
|
||||
|
||||
if err := a.Deactivate(ctx, "acme.feeds"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := registry.Get("acme.feeds/feed"); err == nil {
|
||||
t.Fatal("connector survived Deactivate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginConnectorCannotShadowABuiltin(t *testing.T) {
|
||||
registry := datasource.NewConnectorRegistry()
|
||||
if err := registry.Register(&remoteConnector{typeID: "rss"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registry.RegisterPlugin(&remoteConnector{typeID: "rss"}, datasource.ConnectorMetadata{}); err == nil {
|
||||
t.Fatal("a plugin must not replace a builtin connector")
|
||||
}
|
||||
registry.Unregister("rss")
|
||||
if _, err := registry.Get("rss"); err != nil {
|
||||
t.Fatal("Unregister must not remove builtins")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
package activate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/plugin/install"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/tenancy"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// ClientSource reaches running code plugins (the node's plugin host).
|
||||
type ClientSource interface {
|
||||
Client(pluginID string) (*client.Client, error)
|
||||
}
|
||||
|
||||
// Invoker is how the remote adapters call a code plugin: it finds the
|
||||
// process and builds the protocol envelope from the request context and the
|
||||
// plugin's decrypted system and tenant configuration.
|
||||
type Invoker struct {
|
||||
clients ClientSource
|
||||
|
||||
mu sync.RWMutex
|
||||
tenancy *tenancy.Service
|
||||
plugins interfaces.PluginRepository
|
||||
}
|
||||
|
||||
// NewInvoker creates an Invoker; Bind completes it.
|
||||
func NewInvoker(clients ClientSource) *Invoker { return &Invoker{clients: clients} }
|
||||
|
||||
// Bind supplies the tenant configuration and installed plugin rows.
|
||||
func (iv *Invoker) Bind(t *tenancy.Service, plugins interfaces.PluginRepository) {
|
||||
iv.mu.Lock()
|
||||
iv.tenancy, iv.plugins = t, plugins
|
||||
iv.mu.Unlock()
|
||||
}
|
||||
|
||||
// Default timeouts per call kind, used when the caller set no deadline.
|
||||
const (
|
||||
searchTimeout = 10 * time.Second
|
||||
metadataTimeout = time.Minute
|
||||
)
|
||||
|
||||
// withDefaultTimeout bounds ctx unless it already has a deadline.
|
||||
func withDefaultTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) {
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return context.WithCancel(ctx)
|
||||
}
|
||||
return context.WithTimeout(ctx, d)
|
||||
}
|
||||
|
||||
// Envelope builds the call envelope for a plugin with an instance config.
|
||||
func (iv *Invoker) Envelope(
|
||||
ctx context.Context,
|
||||
m *manifest.Manifest,
|
||||
instance map[string]any,
|
||||
) (pluginapi.Envelope, error) {
|
||||
env := pluginapi.Envelope{Config: pluginapi.Config{Instance: instance}}
|
||||
if tenantID, ok := types.TenantIDFromContext(ctx); ok {
|
||||
env.Context.TenantID = tenantID
|
||||
}
|
||||
if userID, ok := ctx.Value(types.UserIDContextKey).(string); ok {
|
||||
env.Context.UserID = userID
|
||||
}
|
||||
if lang, ok := types.LanguageFromContext(ctx); ok {
|
||||
env.Context.Locale = lang
|
||||
}
|
||||
if rid, ok := ctx.Value(types.RequestIDContextKey).(string); ok {
|
||||
env.Context.RequestID = rid
|
||||
}
|
||||
if d, ok := ctx.Deadline(); ok {
|
||||
env.Context.Deadline = &d
|
||||
}
|
||||
iv.mu.RLock()
|
||||
t, plugins := iv.tenancy, iv.plugins
|
||||
iv.mu.RUnlock()
|
||||
if plugins != nil && len(m.Config.SystemSchema) > 0 {
|
||||
sys, _, err := install.OpenSystemConfig(ctx, plugins, m)
|
||||
if err != nil {
|
||||
return env, err
|
||||
}
|
||||
env.Config.System = sys
|
||||
}
|
||||
if t != nil && env.Context.TenantID != 0 && len(m.Config.TenantSchema) > 0 {
|
||||
cfg, _, err := t.OpenConfig(ctx, env.Context.TenantID, m.ID)
|
||||
if err != nil {
|
||||
return env, err
|
||||
}
|
||||
env.Config.Tenant = cfg
|
||||
}
|
||||
return env, nil
|
||||
}
|
||||
|
||||
// callError presents a plugin's error by its message, the way WeKnora's own
|
||||
// errors read in the UI; the protocol code stays reachable through
|
||||
// errors.As(err, **pluginapi.Error).
|
||||
type callError struct {
|
||||
pluginID string
|
||||
err *pluginapi.Error
|
||||
}
|
||||
|
||||
func (e *callError) Error() string {
|
||||
if e.err.Code == pluginapi.CodeUnavailable {
|
||||
return "plugin " + e.pluginID + " is unavailable: " + e.err.Message
|
||||
}
|
||||
return e.err.Message
|
||||
}
|
||||
|
||||
func (e *callError) Unwrap() error { return e.err }
|
||||
|
||||
func present(pluginID string, err error) error {
|
||||
if pe, ok := pluginapi.AsError(err); ok {
|
||||
return &callError{pluginID: pluginID, err: pe}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Call makes a unary call to a plugin.
|
||||
func (iv *Invoker) Call(
|
||||
ctx context.Context, m *manifest.Manifest, path string, instance map[string]any, input, out any,
|
||||
) error {
|
||||
c, err := iv.clients.Client(m.ID)
|
||||
if err != nil {
|
||||
return present(m.ID, err)
|
||||
}
|
||||
env, err := iv.Envelope(ctx, m, instance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return present(m.ID, c.Call(ctx, path, env, input, out))
|
||||
}
|
||||
|
||||
// Stream makes a streaming call to a plugin.
|
||||
func (iv *Invoker) Stream(
|
||||
ctx context.Context, m *manifest.Manifest, path string, instance map[string]any, input any,
|
||||
fn func(pluginapi.Event) error,
|
||||
) ([]byte, error) {
|
||||
c, err := iv.clients.Client(m.ID)
|
||||
if err != nil {
|
||||
return nil, present(m.ID, err)
|
||||
}
|
||||
env, err := iv.Envelope(ctx, m, instance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
end, err := c.Stream(ctx, path, env, input, fn)
|
||||
return end, present(m.ID, err)
|
||||
}
|
||||
|
|
@ -0,0 +1,278 @@
|
|||
package activate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/configschema"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/pkg"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// WebSearch registers code plugins' web search providers as provider types.
|
||||
// A plugin provider stores its configuration in the same parameters as a
|
||||
// builtin one: the secret api_key and string fields under extra_config, so
|
||||
// encryption, redaction and the credentials endpoint work unchanged.
|
||||
type WebSearch struct {
|
||||
iv *Invoker
|
||||
registry *infra_web_search.Registry
|
||||
|
||||
mu sync.Mutex
|
||||
registered map[string][]string // plugin ID → provider type IDs
|
||||
}
|
||||
|
||||
// NewWebSearch creates the web search activator.
|
||||
func NewWebSearch(iv *Invoker, registry *infra_web_search.Registry) *WebSearch {
|
||||
return &WebSearch{iv: iv, registry: registry, registered: map[string][]string{}}
|
||||
}
|
||||
|
||||
// Name implements reconcile.Activator.
|
||||
func (a *WebSearch) Name() string { return "webSearch" }
|
||||
|
||||
// Activate implements reconcile.Activator: all of a plugin's providers or
|
||||
// none.
|
||||
func (a *WebSearch) Activate(_ context.Context, l *reconcile.Loaded) error {
|
||||
var ids []string
|
||||
for _, c := range l.Manifest.Contributes[manifest.PointWebSearch] {
|
||||
id := manifest.QualifiedID(l.Manifest.ID, c.ID)
|
||||
pt, err := webSearchType(l, c, id)
|
||||
if err == nil {
|
||||
m, local := l.Manifest, c.ID
|
||||
err = a.registry.RegisterPlugin(
|
||||
pt,
|
||||
func(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
|
||||
return &remoteSearch{
|
||||
iv: a.iv,
|
||||
m: m,
|
||||
local: local,
|
||||
typeID: id,
|
||||
instance: webSearchInstance(params),
|
||||
}, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
for _, done := range ids {
|
||||
a.registry.Unregister(done)
|
||||
}
|
||||
return fmt.Errorf("webSearch %s: %w", c.ID, err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.registered[l.Manifest.ID] = ids
|
||||
a.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deactivate implements reconcile.Activator.
|
||||
func (a *WebSearch) Deactivate(_ context.Context, pluginID string) error {
|
||||
a.mu.Lock()
|
||||
ids := a.registered[pluginID]
|
||||
delete(a.registered, pluginID)
|
||||
a.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
a.registry.Unregister(id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// webSearchType describes a plugin provider for the type list.
|
||||
func webSearchType(l *reconcile.Loaded, c manifest.Contribution, id string) (infra_web_search.PluginType, error) {
|
||||
src, err := instanceSchema(l.Package, c)
|
||||
if err != nil {
|
||||
return infra_web_search.PluginType{}, err
|
||||
}
|
||||
schema, requiresKey, err := webSearchSchema(src)
|
||||
if err != nil {
|
||||
return infra_web_search.PluginType{}, err
|
||||
}
|
||||
icon, err := iconDataURI(l.Package, c.Icon)
|
||||
if err != nil {
|
||||
return infra_web_search.PluginType{}, err
|
||||
}
|
||||
info := types.WebSearchProviderTypeInfo{
|
||||
ID: id, Name: c.Name.Default, Names: c.Name.Locales, Description: c.Description.Default,
|
||||
RequiresAPIKey: requiresKey, Icon: icon, PluginID: l.Manifest.ID, ConfigSchema: schema,
|
||||
}
|
||||
validate := func(params types.WebSearchProviderParameters) error {
|
||||
value := map[string]any{"api_key": params.APIKey}
|
||||
extra := map[string]any{}
|
||||
for k, v := range params.ExtraConfig {
|
||||
extra[k] = v
|
||||
}
|
||||
value["extra_config"] = extra
|
||||
if errs := configschema.Validate(schema, value); len(errs) > 0 {
|
||||
return errs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return infra_web_search.PluginType{Info: info, Validate: validate}, nil
|
||||
}
|
||||
|
||||
// webSearchSchema maps a plugin's instance schema onto the provider
|
||||
// parameters: api_key stays top-level (the only secret), everything else
|
||||
// moves under extra_config, which stores strings.
|
||||
func webSearchSchema(src *configschema.Schema) (*configschema.Schema, bool, error) {
|
||||
out := configschema.Object()
|
||||
if src == nil {
|
||||
return out, false, nil
|
||||
}
|
||||
required := map[string]bool{}
|
||||
for _, k := range src.Required {
|
||||
required[k] = true
|
||||
}
|
||||
extra := &configschema.Schema{
|
||||
Type: configschema.TypeObject, Group: "credentials", Order: 40, Properties: map[string]*configschema.Schema{},
|
||||
}
|
||||
var keys []string
|
||||
for k := range src.Properties {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
requiresKey := false
|
||||
for _, k := range keys {
|
||||
prop := *src.Properties[k]
|
||||
if k == "api_key" {
|
||||
prop.Type, prop.Secret, prop.Group = configschema.TypeString, true, "credentials"
|
||||
out.Set("api_key", &prop, required[k])
|
||||
requiresKey = required[k]
|
||||
continue
|
||||
}
|
||||
if prop.Secret {
|
||||
return nil, false, fmt.Errorf("instanceSchema field %s is secret; only api_key may be", k)
|
||||
}
|
||||
if prop.Type != "" && prop.Type != configschema.TypeString {
|
||||
return nil, false, fmt.Errorf(
|
||||
"instanceSchema field %s must be a string (web search settings are stored as strings)",
|
||||
k,
|
||||
)
|
||||
}
|
||||
prop.Type, prop.Group = configschema.TypeString, ""
|
||||
extra.Set(k, &prop, required[k])
|
||||
}
|
||||
if len(extra.Properties) > 0 {
|
||||
out.Set("extra_config", extra, len(extra.Required) > 0)
|
||||
}
|
||||
return out, requiresKey, nil
|
||||
}
|
||||
|
||||
// webSearchInstance is what the plugin receives as config.instance.
|
||||
func webSearchInstance(params types.WebSearchProviderParameters) map[string]any {
|
||||
out := map[string]any{}
|
||||
if params.APIKey != "" {
|
||||
out["api_key"] = params.APIKey
|
||||
}
|
||||
for k, v := range params.ExtraConfig {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// instanceSchema reads a contribution's instanceSchema file (JSON or YAML).
|
||||
func instanceSchema(p *pkg.Package, c manifest.Contribution) (*configschema.Schema, error) {
|
||||
if c.InstanceSchema == "" || p == nil {
|
||||
return nil, nil
|
||||
}
|
||||
raw, ok := p.ReadFile(c.InstanceSchema)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("instanceSchema %s is not in the package", c.InstanceSchema)
|
||||
}
|
||||
var v any
|
||||
if err := yaml.Unmarshal(raw, &v); err != nil {
|
||||
return nil, fmt.Errorf("instanceSchema %s: %w", c.InstanceSchema, err)
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := configschema.Parse(b)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("instanceSchema %s: %w", c.InstanceSchema, err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// maxIconBytes caps an icon inlined into type listings.
|
||||
const maxIconBytes = 64 << 10
|
||||
|
||||
// iconDataURI inlines a package icon (SVG or PNG) as a data: URI.
|
||||
func iconDataURI(p *pkg.Package, file string) (string, error) {
|
||||
if file == "" || p == nil {
|
||||
return "", nil
|
||||
}
|
||||
b, ok := p.ReadFile(file)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("icon %s is not in the package", file)
|
||||
}
|
||||
if len(b) > maxIconBytes {
|
||||
return "", fmt.Errorf("icon %s is over %d bytes", file, maxIconBytes)
|
||||
}
|
||||
var mime string
|
||||
switch strings.ToLower(path.Ext(file)) {
|
||||
case ".svg":
|
||||
mime = "image/svg+xml"
|
||||
case ".png":
|
||||
mime = "image/png"
|
||||
default:
|
||||
return "", fmt.Errorf("icon %s must be .svg or .png", file)
|
||||
}
|
||||
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// remoteSearch is one configured plugin provider.
|
||||
type remoteSearch struct {
|
||||
iv *Invoker
|
||||
m *manifest.Manifest
|
||||
local string
|
||||
typeID string
|
||||
instance map[string]any
|
||||
}
|
||||
|
||||
// Name implements interfaces.WebSearchProvider.
|
||||
func (r *remoteSearch) Name() string { return r.typeID }
|
||||
|
||||
// Search implements interfaces.WebSearchProvider.
|
||||
func (r *remoteSearch) Search(
|
||||
ctx context.Context, query string, maxResults int, includeDate bool,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
return r.SearchWithFilters(ctx, query, maxResults, includeDate, types.WebSearchFilters{})
|
||||
}
|
||||
|
||||
// SearchWithFilters implements interfaces.FilteredWebSearchProvider; the
|
||||
// plugin must fail rather than ignore a filter it cannot honour.
|
||||
func (r *remoteSearch) SearchWithFilters(
|
||||
ctx context.Context, query string, maxResults int, includeDate bool, filters types.WebSearchFilters,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
ctx, cancel := withDefaultTimeout(ctx, searchTimeout)
|
||||
defer cancel()
|
||||
in := pluginapi.SearchInput{
|
||||
Query: query, MaxResults: maxResults, IncludeDate: includeDate,
|
||||
Region: filters.Country, Freshness: filters.Freshness,
|
||||
}
|
||||
var out pluginapi.SearchOutput
|
||||
if err := r.iv.Call(ctx, r.m, pluginapi.SearchPath(r.local), r.instance, in, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results := make([]*types.WebSearchResult, 0, len(out.Results))
|
||||
for _, h := range out.Results {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: h.Title, URL: h.URL, Snippet: h.Snippet, Content: h.Content,
|
||||
Source: r.typeID, Age: h.Age, PublishedAt: h.PublishedAt,
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
package activate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/pkg"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/plugintest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
const searchPluginManifest = `schemaVersion: 1
|
||||
id: acme.search
|
||||
version: 1.0.0
|
||||
apiVersion: weknora.plugin/v1
|
||||
name: { en-US: ACME Search }
|
||||
publisher: { id: acme }
|
||||
runtime: { type: host, kind: binary, entry: bin/search }
|
||||
contributes:
|
||||
webSearch:
|
||||
- id: brave
|
||||
name: { en-US: ACME Brave, zh-CN: ACME 勇敢搜索 }
|
||||
icon: icons/brave.svg
|
||||
instanceSchema: schemas/brave.yaml
|
||||
`
|
||||
|
||||
// fakeClients serves one plugin from an in-process SDK handler.
|
||||
type fakeClients struct{ c *client.Client }
|
||||
|
||||
func (f fakeClients) Client(string) (*client.Client, error) { return f.c, nil }
|
||||
|
||||
func searchLoaded(t *testing.T, schema string) *reconcile.Loaded {
|
||||
t.Helper()
|
||||
p, err := pkg.Open(plugintest.Zip(t, map[string]string{
|
||||
"plugin.yaml": searchPluginManifest,
|
||||
"bin/search": "x",
|
||||
"icons/brave.svg": `<svg xmlns="http://www.w3.org/2000/svg"/>`,
|
||||
"schemas/brave.yaml": schema,
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &reconcile.Loaded{Manifest: p.Manifest, Package: p}
|
||||
}
|
||||
|
||||
const braveSchema = `type: object
|
||||
required: [api_key, market]
|
||||
properties:
|
||||
api_key: { type: string, title: API key }
|
||||
market: { type: string, enum: [en-US, zh-CN], default: en-US }
|
||||
`
|
||||
|
||||
func TestPluginWebSearchProvider(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
plugin := pluginsdk.New(pluginsdk.Info{ID: "acme.search", Version: "1.0.0"})
|
||||
plugin.WebSearch(
|
||||
"brave",
|
||||
pluginsdk.WebSearchFunc(
|
||||
func(_ context.Context, call *pluginsdk.Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
|
||||
if call.Config.Instance["api_key"] != "k-1" || call.Config.Instance["market"] != "zh-CN" {
|
||||
return nil, pluginapi.Errorf(
|
||||
pluginapi.CodeUnauthorized,
|
||||
"bad instance config %v",
|
||||
call.Config.Instance,
|
||||
)
|
||||
}
|
||||
if in.Region != "" && in.Region != "CN" {
|
||||
return nil, pluginapi.Errorf(pluginapi.CodeInvalidConfig, "region %s unsupported", in.Region)
|
||||
}
|
||||
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{
|
||||
Title: in.Query, URL: fmt.Sprint("https://acme.example/?tenant=", call.TenantID),
|
||||
}}}, nil
|
||||
},
|
||||
),
|
||||
)
|
||||
srv := httptest.NewServer(plugin.Handler())
|
||||
defer srv.Close()
|
||||
|
||||
registry := infra_web_search.NewRegistry()
|
||||
a := NewWebSearch(NewInvoker(fakeClients{client.New(srv.URL, nil, nil)}), registry)
|
||||
if err := a.Activate(ctx, searchLoaded(t, braveSchema)); err != nil {
|
||||
t.Fatalf("Activate: %v", err)
|
||||
}
|
||||
|
||||
pt, ok := registry.PluginType("acme.search/brave")
|
||||
if !ok {
|
||||
t.Fatal("type not registered")
|
||||
}
|
||||
info := pt.Info
|
||||
if info.Names["zh-CN"] != "ACME 勇敢搜索" || !info.RequiresAPIKey ||
|
||||
!strings.HasPrefix(info.Icon, "data:image/svg+xml;base64,") ||
|
||||
info.PluginID != "acme.search" {
|
||||
t.Fatalf("info = %+v", info)
|
||||
}
|
||||
if ak := info.ConfigSchema.Properties["api_key"]; ak == nil || !ak.Secret || ak.Group != "credentials" {
|
||||
t.Fatalf("api_key schema = %+v", ak)
|
||||
}
|
||||
if ec := info.ConfigSchema.Properties["extra_config"]; ec == nil || ec.Properties["market"] == nil {
|
||||
t.Fatalf("extra_config schema = %+v", ec)
|
||||
}
|
||||
if err := pt.Validate(types.WebSearchProviderParameters{APIKey: "k"}); err == nil {
|
||||
t.Fatal("a missing required extra_config field must fail validation")
|
||||
}
|
||||
params := types.WebSearchProviderParameters{APIKey: "k-1", ExtraConfig: map[string]string{"market": "zh-CN"}}
|
||||
if err := pt.Validate(params); err != nil {
|
||||
t.Fatalf("valid params rejected: %v", err)
|
||||
}
|
||||
|
||||
provider, err := registry.CreateProvider("acme.search/brave", params)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tctx := context.WithValue(ctx, types.TenantIDContextKey, uint64(7))
|
||||
res, err := provider.Search(tctx, "weknora", 3, false)
|
||||
if err != nil || len(res) != 1 || res[0].Title != "weknora" || res[0].URL != "https://acme.example/?tenant=7" ||
|
||||
res[0].Source != "acme.search/brave" {
|
||||
t.Fatalf("search = %+v, %v", res, err)
|
||||
}
|
||||
filtered := provider.(interfaces.FilteredWebSearchProvider)
|
||||
if _, err := filtered.SearchWithFilters(tctx, "q", 3, false, types.WebSearchFilters{Country: "DE"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "region DE unsupported") {
|
||||
t.Fatalf("an unsupported filter must fail, got %v", err)
|
||||
}
|
||||
|
||||
if err := a.Deactivate(ctx, "acme.search"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := registry.PluginType("acme.search/brave"); ok {
|
||||
t.Fatal("type survived Deactivate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginWebSearchSchemaRules(t *testing.T) {
|
||||
a := NewWebSearch(NewInvoker(fakeClients{}), infra_web_search.NewRegistry())
|
||||
for name, schema := range map[string]string{
|
||||
"secret field": "type: object\nproperties: { token: { type: string, x-secret: true } }\n",
|
||||
"number field": "type: object\nproperties: { limit: { type: integer } }\n",
|
||||
} {
|
||||
if err := a.Activate(context.Background(), searchLoaded(t, schema)); err == nil {
|
||||
t.Errorf("%s: want an activation error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
// Package host is the embedded plugin host: it runs host-runtime plugins as
|
||||
// child processes of this WeKnora node and hands out clients to reach them.
|
||||
// Each node runs its own copy of every enabled host plugin (the reconciler
|
||||
// activates it everywhere), so a call never leaves the node.
|
||||
//
|
||||
// A plugin process gets a private socket, a random token, an environment
|
||||
// built from scratch (no WeKnora secrets) and an egress proxy limited to the
|
||||
// hosts its manifest was granted. The host checks that the process is the
|
||||
// installed package, health-checks it and restarts it with backoff.
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// StateReporter learns when a running plugin's health changes, so the
|
||||
// cluster view shows a crashed plugin as degraded.
|
||||
type StateReporter interface {
|
||||
ReportRuntime(pluginID string, healthy bool, err error)
|
||||
}
|
||||
|
||||
// Manager runs host plugins. It is a reconcile.Activator and must come
|
||||
// before the activators that route calls to the plugins.
|
||||
type Manager struct {
|
||||
reporter StateReporter
|
||||
|
||||
mu sync.Mutex
|
||||
procs map[string]*process
|
||||
}
|
||||
|
||||
// NewManager creates an empty host.
|
||||
func NewManager() *Manager { return &Manager{procs: map[string]*process{}} }
|
||||
|
||||
// SetReporter wires health changes to the reconciler.
|
||||
func (m *Manager) SetReporter(r StateReporter) {
|
||||
m.mu.Lock()
|
||||
m.reporter = r
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// Name implements reconcile.Activator.
|
||||
func (m *Manager) Name() string { return "host" }
|
||||
|
||||
// ActivatesInPlace implements reconcile.InPlaceActivator: an upgrade starts
|
||||
// the new process before the old one stops.
|
||||
func (m *Manager) ActivatesInPlace() {}
|
||||
|
||||
// Activate starts a host plugin and returns once it is ready. Other runtimes
|
||||
// are ignored. A new version replaces the old process only after it started.
|
||||
func (m *Manager) Activate(ctx context.Context, l *reconcile.Loaded) error {
|
||||
if l.Manifest.Runtime.Type != manifest.RuntimeHost {
|
||||
return nil
|
||||
}
|
||||
if l.Manifest.Runtime.Kind != "binary" {
|
||||
return fmt.Errorf("runtime.kind %q is not supported by this host yet; only binary", l.Manifest.Runtime.Kind)
|
||||
}
|
||||
id := l.Manifest.ID
|
||||
p, err := startProcess(spec{m: l.Manifest, dir: l.Dir}, func(s State, err error) {
|
||||
m.report(id, s, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.mu.Lock()
|
||||
old := m.procs[id]
|
||||
m.procs[id] = p
|
||||
m.mu.Unlock()
|
||||
if old != nil {
|
||||
old.stop()
|
||||
}
|
||||
logger.Infof(ctx, "[plugin] host started %s %s", id, l.Manifest.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deactivate implements reconcile.Activator: it stops the process.
|
||||
func (m *Manager) Deactivate(ctx context.Context, pluginID string) error {
|
||||
m.mu.Lock()
|
||||
p := m.procs[pluginID]
|
||||
delete(m.procs, pluginID)
|
||||
m.mu.Unlock()
|
||||
if p != nil {
|
||||
p.stop()
|
||||
logger.Infof(ctx, "[plugin] host stopped %s", pluginID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) report(pluginID string, s State, err error) {
|
||||
m.mu.Lock()
|
||||
r := m.reporter
|
||||
m.mu.Unlock()
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
switch s {
|
||||
case StateReady:
|
||||
r.ReportRuntime(pluginID, true, nil)
|
||||
case StateDegraded:
|
||||
r.ReportRuntime(pluginID, false, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Client returns a client for a running plugin. The error is a retryable
|
||||
// pluginapi unavailable error while the plugin is starting or restarting.
|
||||
func (m *Manager) Client(pluginID string) (*client.Client, error) {
|
||||
m.mu.Lock()
|
||||
p := m.procs[pluginID]
|
||||
m.mu.Unlock()
|
||||
if p == nil {
|
||||
return nil, &pluginapi.Error{
|
||||
Code: pluginapi.CodeUnavailable, Message: fmt.Sprintf("plugin %s is not running on this node", pluginID),
|
||||
}
|
||||
}
|
||||
return p.Client()
|
||||
}
|
||||
|
||||
// Close stops every plugin, for shutdown.
|
||||
func (m *Manager) Close() {
|
||||
m.mu.Lock()
|
||||
procs := m.procs
|
||||
m.procs = map[string]*process{}
|
||||
m.mu.Unlock()
|
||||
var wg sync.WaitGroup
|
||||
for _, p := range procs {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
p.stop()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
var (
|
||||
buildOnce sync.Once
|
||||
binary string
|
||||
buildErr error
|
||||
)
|
||||
|
||||
// echoBinary builds testdata/echoplugin once per test run.
|
||||
func echoBinary(t *testing.T) string {
|
||||
t.Helper()
|
||||
buildOnce.Do(func() {
|
||||
dir, err := os.MkdirTemp("", "echoplugin")
|
||||
if err != nil {
|
||||
buildErr = err
|
||||
return
|
||||
}
|
||||
binary = filepath.Join(dir, "echo")
|
||||
out, err := exec.Command("go", "build", "-o", binary, "./testdata/echoplugin").CombinedOutput()
|
||||
if err != nil {
|
||||
buildErr = err
|
||||
t.Logf("%s", out)
|
||||
}
|
||||
})
|
||||
if buildErr != nil {
|
||||
t.Fatalf("build echo plugin: %v", buildErr)
|
||||
}
|
||||
return binary
|
||||
}
|
||||
|
||||
// install lays out an extracted package with the echo binary.
|
||||
func install(t *testing.T, version string, fakeVersion string) *reconcile.Loaded {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "bin", runtime.GOOS+"-"+runtime.GOARCH)
|
||||
if err := os.MkdirAll(bin, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(echoBinary(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Extraction writes plain files; the host must make the entry runnable.
|
||||
name := "echo"
|
||||
if runtime.GOOS == "windows" {
|
||||
name += ".exe"
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(bin, name), data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fakeVersion != "" {
|
||||
_ = os.WriteFile(filepath.Join(dir, "version"), []byte(fakeVersion), 0o644)
|
||||
}
|
||||
m := &manifest.Manifest{
|
||||
SchemaVersion: manifest.SchemaVersion, ID: "acme.echo", Version: version, APIVersion: pluginapi.APIVersion,
|
||||
Name: manifest.Text("Echo", nil), Publisher: manifest.Publisher{ID: "acme"},
|
||||
Runtime: manifest.Runtime{Type: manifest.RuntimeHost, Kind: "binary", Entry: "bin/{os}-{arch}/echo"},
|
||||
Contributes: manifest.Contributions{
|
||||
manifest.PointWebSearch: {{ID: "echo", Name: manifest.Text("Echo", nil)}},
|
||||
},
|
||||
}
|
||||
return &reconcile.Loaded{Manifest: m, Dir: dir}
|
||||
}
|
||||
|
||||
type reports struct {
|
||||
mu sync.Mutex
|
||||
log []string
|
||||
}
|
||||
|
||||
func (r *reports) ReportRuntime(id string, healthy bool, _ error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.log = append(r.log, id+":"+strconv.FormatBool(healthy))
|
||||
}
|
||||
|
||||
func (r *reports) String() string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return strings.Join(r.log, ",")
|
||||
}
|
||||
|
||||
func fastTimings(t *testing.T) {
|
||||
t.Helper()
|
||||
oldBackoff, oldInterval, oldGrace := restartBackoffFloor, healthInterval, stopGrace
|
||||
restartBackoffFloor, healthInterval, stopGrace = 20*time.Millisecond, 100*time.Millisecond, 2*time.Second
|
||||
t.Cleanup(func() { restartBackoffFloor, healthInterval, stopGrace = oldBackoff, oldInterval, oldGrace })
|
||||
}
|
||||
|
||||
func search(t *testing.T, m *Manager, q string) (string, error) {
|
||||
t.Helper()
|
||||
c, err := m.Client("acme.echo")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out pluginapi.SearchOutput
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
err = c.Call(ctx, pluginapi.SearchPath("echo"), pluginapi.Envelope{}, pluginapi.SearchInput{Query: q}, &out)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return out.Results[0].Title, nil
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostRunsRestartsAndStopsAPlugin(t *testing.T) {
|
||||
fastTimings(t)
|
||||
t.Setenv("DB_PASSWORD", "must-not-leak")
|
||||
ctx := context.Background()
|
||||
m := NewManager()
|
||||
rep := &reports{}
|
||||
m.SetReporter(rep)
|
||||
defer m.Close()
|
||||
|
||||
if err := m.Activate(ctx, install(t, "1.0.0", "")); err != nil {
|
||||
t.Fatalf("Activate: %v", err)
|
||||
}
|
||||
if got, err := search(t, m, "hello"); err != nil || got != "hello" {
|
||||
t.Fatalf("search = %q, %v", got, err)
|
||||
}
|
||||
env, err := search(t, m, "env")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(env, "DB_PASSWORD") || !strings.Contains(env, "HTTPS_PROXY") ||
|
||||
!strings.Contains(env, pluginapi.EnvToken) {
|
||||
t.Fatalf("plugin environment = %s", env)
|
||||
}
|
||||
|
||||
pid1, _ := search(t, m, "pid")
|
||||
_, _ = search(t, m, "crash")
|
||||
waitFor(t, "restart", func() bool {
|
||||
pid, err := search(t, m, "pid")
|
||||
return err == nil && pid != pid1
|
||||
})
|
||||
if !strings.Contains(rep.String(), "acme.echo:false") || !strings.HasSuffix(rep.String(), "acme.echo:true") {
|
||||
t.Fatalf("reports = %s", rep)
|
||||
}
|
||||
|
||||
pid2, _ := search(t, m, "pid")
|
||||
if err := m.Deactivate(ctx, "acme.echo"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.Client("acme.echo"); err == nil {
|
||||
t.Fatal("a stopped plugin must have no client")
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
n, _ := strconv.Atoi(pid2)
|
||||
if proc, err := os.FindProcess(n); err == nil && proc.Signal(syscall.Signal(0)) == nil {
|
||||
t.Fatalf("process %d still runs after Deactivate", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostRefusesAProcessThatIsNotThePackage(t *testing.T) {
|
||||
fastTimings(t)
|
||||
m := NewManager()
|
||||
defer m.Close()
|
||||
err := m.Activate(context.Background(), install(t, "1.0.0", "9.9.9"))
|
||||
if err == nil || !strings.Contains(err.Error(), "reports acme.echo@9.9.9") {
|
||||
t.Fatalf("want a manifest mismatch, got %v", err)
|
||||
}
|
||||
l := install(t, "1.0.0", "")
|
||||
l.Manifest.Contributes[manifest.PointWebSearch] = append(l.Manifest.Contributes[manifest.PointWebSearch],
|
||||
manifest.Contribution{ID: "missing", Name: manifest.Text("Missing", nil)})
|
||||
if err := m.Activate(context.Background(), l); err == nil || !strings.Contains(err.Error(), "webSearch/missing") {
|
||||
t.Fatalf("want a missing contribution error, got %v", err)
|
||||
}
|
||||
l = install(t, "1.0.0", "")
|
||||
l.Manifest.Runtime.Entry = "bin/{os}-{arch}/nope"
|
||||
if err := m.Activate(context.Background(), l); err == nil || !strings.Contains(err.Error(), "has no") {
|
||||
t.Fatalf("want a missing build error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeSwapsProcessesInPlace(t *testing.T) {
|
||||
fastTimings(t)
|
||||
ctx := context.Background()
|
||||
m := NewManager()
|
||||
defer m.Close()
|
||||
if err := m.Activate(ctx, install(t, "1.0.0", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid1, _ := search(t, m, "pid")
|
||||
if err := m.Activate(ctx, install(t, "1.1.0", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid2, err := search(t, m, "pid")
|
||||
if err != nil || pid2 == pid1 {
|
||||
t.Fatalf("after upgrade pid = %s (was %s), %v", pid2, pid1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEgressProxyEnforcesThePolicy(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("hello from " + r.Host))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
p, err := startEgressProxy("acme.echo", []string{"api.allowed.test", "*.wild.test"}, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer p.Close()
|
||||
// Every name resolves to the test server; the policy is what differs.
|
||||
p.dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, upstream.Listener.Addr().String())
|
||||
}
|
||||
proxyURL, _ := url.Parse(p.URL())
|
||||
c := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
|
||||
for host, want := range map[string]int{
|
||||
"api.allowed.test": 200, "x.wild.test": 200, "wild.test": 403, "evil.test": 403,
|
||||
} {
|
||||
resp, err := c.Get("http://" + host + "/")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", host, err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != want {
|
||||
t.Errorf("%s: status %d, want %d", host, resp.StatusCode, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEgressProxyTunnelsHTTPS(t *testing.T) {
|
||||
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("secure"))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
p, err := startEgressProxy("acme.echo", []string{"*.wild.test"}, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer p.Close()
|
||||
p.dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, upstream.Listener.Addr().String())
|
||||
}
|
||||
proxyURL, _ := url.Parse(p.URL())
|
||||
tr := upstream.Client().Transport.(*http.Transport).Clone()
|
||||
tr.Proxy = http.ProxyURL(proxyURL)
|
||||
tr.TLSClientConfig.InsecureSkipVerify = true
|
||||
c := &http.Client{Transport: tr}
|
||||
|
||||
resp, err := c.Get("https://api.wild.test/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := make([]byte, 16)
|
||||
n, _ := resp.Body.Read(body)
|
||||
_ = resp.Body.Close()
|
||||
if string(body[:n]) != "secure" {
|
||||
t.Fatalf("tunnelled body = %q", body[:n])
|
||||
}
|
||||
if _, err := c.Get("https://evil.test/"); err == nil || !strings.Contains(err.Error(), "Forbidden") {
|
||||
t.Fatalf("CONNECT to a host outside the policy must be refused, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolVersionsAgree(t *testing.T) {
|
||||
if manifest.ExtensionAPIVersion != pluginapi.APIVersion {
|
||||
t.Fatalf("manifest accepts %q, the protocol package speaks %q",
|
||||
manifest.ExtensionAPIVersion, pluginapi.APIVersion)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
//go:build linux
|
||||
|
||||
package host
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// configureChild puts the plugin in its own process group and has the kernel
|
||||
// kill it if WeKnora dies, so a crash never leaves orphaned plugins.
|
||||
func configureChild(cmd *exec.Cmd) {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
|
||||
}
|
||||
|
||||
func terminate(cmd *exec.Cmd) error { return cmd.Process.Signal(syscall.SIGTERM) }
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
//go:build unix && !linux
|
||||
|
||||
package host
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// configureChild puts the plugin in its own process group, so a signal
|
||||
// meant for WeKnora's terminal does not reach it directly.
|
||||
func configureChild(cmd *exec.Cmd) {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
}
|
||||
|
||||
func terminate(cmd *exec.Cmd) error { return cmd.Process.Signal(syscall.SIGTERM) }
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
//go:build windows
|
||||
|
||||
package host
|
||||
|
||||
import "os/exec"
|
||||
|
||||
func configureChild(*exec.Cmd) {}
|
||||
|
||||
// terminate kills the plugin: Windows has no SIGTERM for console-less
|
||||
// children.
|
||||
func terminate(cmd *exec.Cmd) error { return cmd.Process.Kill() }
|
||||
|
|
@ -0,0 +1,438 @@
|
|||
package host
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Timing of a host plugin's life. Variables so tests can shrink them.
|
||||
var (
|
||||
handshakeTimeout = 30 * time.Second
|
||||
healthInterval = 15 * time.Second
|
||||
healthTimeout = 5 * time.Second
|
||||
unhealthyLimit = 3
|
||||
stopGrace = 10 * time.Second
|
||||
restartBackoffFloor = time.Second
|
||||
restartBackoffCeiling = time.Minute
|
||||
)
|
||||
|
||||
// State of a supervised process.
|
||||
type State string
|
||||
|
||||
// Process states reported to the reconciler.
|
||||
const (
|
||||
StateStarting State = "starting"
|
||||
StateReady State = "ready"
|
||||
StateDegraded State = "degraded"
|
||||
StateStopped State = "stopped"
|
||||
)
|
||||
|
||||
// spec is everything needed to start one plugin version.
|
||||
type spec struct {
|
||||
m *manifest.Manifest
|
||||
dir string // extracted package
|
||||
}
|
||||
|
||||
// entryPath resolves runtime.entry for this machine inside the package.
|
||||
func entryPath(m *manifest.Manifest, dir string) (string, error) {
|
||||
rel := strings.NewReplacer("{os}", runtime.GOOS, "{arch}", runtime.GOARCH).Replace(m.Runtime.Entry)
|
||||
if runtime.GOOS == "windows" && filepath.Ext(rel) == "" {
|
||||
rel += ".exe"
|
||||
}
|
||||
p, err := utils.SafeJoinUnderBase(dir, rel)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("runtime.entry %q: %w", m.Runtime.Entry, err)
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the package has no %s build (%s)", runtime.GOOS+"/"+runtime.GOARCH, rel)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", fmt.Errorf("runtime.entry %s is not a file", rel)
|
||||
}
|
||||
// Packages are extracted without modes; the entry has to be executable.
|
||||
if err := os.Chmod(p, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// process is one running plugin version and its supervisor.
|
||||
type process struct {
|
||||
spec spec
|
||||
onState func(State, error)
|
||||
|
||||
mu sync.RWMutex
|
||||
client *client.Client
|
||||
state State
|
||||
lastErr error
|
||||
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
proxy *egressProxy
|
||||
sockDir string
|
||||
}
|
||||
|
||||
// launched is one started child.
|
||||
type launched struct {
|
||||
cmd *exec.Cmd
|
||||
client *client.Client
|
||||
exited chan error
|
||||
}
|
||||
|
||||
// startProcess starts the plugin, waits until it is ready and keeps it
|
||||
// running until stop. It returns once the first start succeeded or failed.
|
||||
func startProcess(sp spec, onState func(State, error)) (*process, error) {
|
||||
entry, err := entryPath(sp.m, sp.dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sockDir, err := socketDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proxy, err := startEgressProxy(sp.m.ID, sp.m.Permissions.Egress, func(f string, a ...any) {
|
||||
logger.Warnf(context.Background(), f, a...)
|
||||
})
|
||||
if err != nil {
|
||||
_ = os.RemoveAll(sockDir)
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
p := &process{spec: sp, onState: onState, cancel: cancel, done: make(chan struct{}), proxy: proxy, sockDir: sockDir}
|
||||
p.setState(StateStarting, nil)
|
||||
first, err := p.launch(ctx, entry)
|
||||
if err != nil {
|
||||
cancel()
|
||||
proxy.Close()
|
||||
_ = os.RemoveAll(sockDir)
|
||||
close(p.done)
|
||||
p.setState(StateStopped, err)
|
||||
return nil, err
|
||||
}
|
||||
// Ready before Activate returns: callers route to the plugin right away.
|
||||
p.setClient(first.client)
|
||||
p.setState(StateReady, nil)
|
||||
go p.supervise(ctx, entry, first)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// socketDir makes a short private directory for the socket: unix socket
|
||||
// paths are limited to about 100 bytes.
|
||||
func socketDir() (string, error) {
|
||||
base := os.Getenv("WEKNORA_PLUGIN_SOCKET_DIR")
|
||||
if base == "" {
|
||||
base = os.TempDir()
|
||||
}
|
||||
return os.MkdirTemp(base, "wkp")
|
||||
}
|
||||
|
||||
func randomToken() string {
|
||||
b := make([]byte, 32)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// childEnv is the plugin's whole environment. It is built from scratch so
|
||||
// WeKnora's own secrets (database, AES key, vendor keys) never reach plugin
|
||||
// code.
|
||||
func (p *process) childEnv(socket, token string) []string {
|
||||
env := []string{
|
||||
pluginapi.EnvSocket + "=" + socket,
|
||||
pluginapi.EnvNetwork + "=unix",
|
||||
pluginapi.EnvToken + "=" + token,
|
||||
pluginapi.EnvPluginID + "=" + p.spec.m.ID,
|
||||
pluginapi.EnvPluginVersion + "=" + p.spec.m.Version,
|
||||
"HOME=" + p.sockDir,
|
||||
"TMPDIR=" + p.sockDir,
|
||||
"HTTP_PROXY=" + p.proxy.URL(),
|
||||
"HTTPS_PROXY=" + p.proxy.URL(),
|
||||
"http_proxy=" + p.proxy.URL(),
|
||||
"https_proxy=" + p.proxy.URL(),
|
||||
"NO_PROXY=",
|
||||
"no_proxy=",
|
||||
}
|
||||
for _, k := range []string{"PATH", "LANG", "LC_ALL", "TZ", "SYSTEMROOT", "WINDIR"} {
|
||||
if v, ok := os.LookupEnv(k); ok {
|
||||
env = append(env, k+"="+v)
|
||||
}
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// launch starts the child and waits for handshake, health and a manifest
|
||||
// that matches the installed package.
|
||||
func (p *process) launch(ctx context.Context, entry string) (*launched, error) {
|
||||
socket := filepath.Join(p.sockDir, "p.sock")
|
||||
_ = os.Remove(socket)
|
||||
token := randomToken()
|
||||
cmd := exec.Command(entry)
|
||||
cmd.Dir = p.spec.dir
|
||||
cmd.Env = p.childEnv(socket, token)
|
||||
configureChild(cmd)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("start %s: %w", filepath.Base(entry), err)
|
||||
}
|
||||
id := p.spec.m.ID + "@" + p.spec.m.Version
|
||||
handshake := make(chan pluginapi.Handshake, 1)
|
||||
hsErr := make(chan error, 1)
|
||||
go forwardLogs(stderr, id, nil, nil)
|
||||
go forwardLogs(stdout, id, handshake, hsErr)
|
||||
exited := make(chan error, 1)
|
||||
go func() { exited <- cmd.Wait() }()
|
||||
|
||||
kill := func() {
|
||||
stopChild(cmd, exited)
|
||||
}
|
||||
timer := time.NewTimer(handshakeTimeout)
|
||||
defer timer.Stop()
|
||||
var hs pluginapi.Handshake
|
||||
select {
|
||||
case hs = <-handshake:
|
||||
case err := <-hsErr:
|
||||
kill()
|
||||
return nil, err
|
||||
case err := <-exited:
|
||||
exited <- err // keep it for supervise
|
||||
return nil, fmt.Errorf("plugin exited before it was ready: %v", err)
|
||||
case <-timer.C:
|
||||
kill()
|
||||
return nil, fmt.Errorf("plugin did not print its handshake within %s", handshakeTimeout)
|
||||
case <-ctx.Done():
|
||||
kill()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if hs.Network == "unix" && hs.Address != socket {
|
||||
kill()
|
||||
return nil, fmt.Errorf("plugin listens on %s, not the socket it was given", hs.Address)
|
||||
}
|
||||
c := client.ForHandshake(hs, client.Bearer(token))
|
||||
cctx, cancel := context.WithTimeout(ctx, healthTimeout)
|
||||
defer cancel()
|
||||
if err := c.Health(cctx); err != nil {
|
||||
c.Close()
|
||||
kill()
|
||||
return nil, fmt.Errorf("plugin is not healthy: %w", err)
|
||||
}
|
||||
m, err := c.Manifest(cctx)
|
||||
if err != nil {
|
||||
c.Close()
|
||||
kill()
|
||||
return nil, fmt.Errorf("read plugin manifest: %w", err)
|
||||
}
|
||||
if err := checkServedManifest(p.spec.m, m); err != nil {
|
||||
c.Close()
|
||||
kill()
|
||||
return nil, err
|
||||
}
|
||||
return &launched{cmd: cmd, client: c, exited: exited}, nil
|
||||
}
|
||||
|
||||
// checkServedManifest makes sure the process is the package that was
|
||||
// installed and serves what the manifest promised.
|
||||
func checkServedManifest(want *manifest.Manifest, got *pluginapi.Manifest) error {
|
||||
if got.ID != want.ID || got.Version != want.Version {
|
||||
return fmt.Errorf("process reports %s@%s, the package is %s@%s", got.ID, got.Version, want.ID, want.Version)
|
||||
}
|
||||
if got.APIVersion != pluginapi.APIVersion {
|
||||
return fmt.Errorf("process speaks %q, this WeKnora speaks %q", got.APIVersion, pluginapi.APIVersion)
|
||||
}
|
||||
for point, contribs := range want.Contributes {
|
||||
if info, ok := manifest.LookupPoint(point); !ok || info.Declarative {
|
||||
continue
|
||||
}
|
||||
served := map[string]bool{}
|
||||
for _, id := range got.Contributes[string(point)] {
|
||||
served[id] = true
|
||||
}
|
||||
for _, c := range contribs {
|
||||
if !served[c.ID] {
|
||||
return fmt.Errorf("plugin.yaml declares %s/%s but the process does not serve it", point, c.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forwardLogs copies a child's output into WeKnora's logs. On stdout it
|
||||
// also watches for the handshake line.
|
||||
func forwardLogs(r io.Reader, id string, handshake chan<- pluginapi.Handshake, hsErr chan<- error) {
|
||||
sc := bufio.NewScanner(r)
|
||||
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
|
||||
seen := false
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if handshake != nil && !seen {
|
||||
hs, ok, err := pluginapi.ParseHandshake(line)
|
||||
if ok {
|
||||
seen = true
|
||||
if err != nil {
|
||||
hsErr <- err
|
||||
} else {
|
||||
handshake <- hs
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
logger.Infof(context.Background(), "[plugin %s] %s", id, line)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *process) setState(s State, err error) {
|
||||
p.mu.Lock()
|
||||
changed := p.state != s || (err != nil) != (p.lastErr != nil)
|
||||
p.state, p.lastErr = s, err
|
||||
p.mu.Unlock()
|
||||
if changed && p.onState != nil {
|
||||
p.onState(s, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *process) setClient(c *client.Client) {
|
||||
p.mu.Lock()
|
||||
old := p.client
|
||||
p.client = c
|
||||
p.mu.Unlock()
|
||||
if old != nil && old != c {
|
||||
old.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// Client returns the client of the running child, or an unavailable error.
|
||||
func (p *process) Client() (*client.Client, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
if p.client == nil || p.state != StateReady {
|
||||
msg := fmt.Sprintf("plugin %s is %s", p.spec.m.ID, p.state)
|
||||
if p.lastErr != nil {
|
||||
msg += ": " + p.lastErr.Error()
|
||||
}
|
||||
return nil, &pluginapi.Error{Code: pluginapi.CodeUnavailable, Message: msg, Retryable: true}
|
||||
}
|
||||
return p.client, nil
|
||||
}
|
||||
|
||||
// supervise keeps the plugin running: health checks, restart with backoff
|
||||
// after a crash or repeated failed checks.
|
||||
func (p *process) supervise(ctx context.Context, entry string, cur *launched) {
|
||||
defer close(p.done)
|
||||
backoff := restartBackoffFloor
|
||||
for {
|
||||
p.setClient(cur.client)
|
||||
p.setState(StateReady, nil)
|
||||
exitErr := p.watch(ctx, cur)
|
||||
p.setClient(nil)
|
||||
if ctx.Err() != nil {
|
||||
p.setState(StateStopped, nil)
|
||||
return
|
||||
}
|
||||
logger.Warnf(ctx, "[plugin] %s stopped unexpectedly: %v; restarting in %s", p.spec.m.ID, exitErr, backoff)
|
||||
p.setState(StateDegraded, exitErr)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.setState(StateStopped, nil)
|
||||
return
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
backoff = min(backoff*2, restartBackoffCeiling)
|
||||
next, err := p.launch(ctx, entry)
|
||||
if err == nil {
|
||||
cur = next
|
||||
backoff = restartBackoffFloor
|
||||
logger.Infof(ctx, "[plugin] %s restarted", p.spec.m.ID)
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
p.setState(StateStopped, nil)
|
||||
return
|
||||
}
|
||||
logger.Warnf(ctx, "[plugin] %s restart failed: %v; next try in %s", p.spec.m.ID, err, backoff)
|
||||
p.setState(StateDegraded, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watch returns when the child exits, fails its health checks, or ctx ends
|
||||
// (then it stops the child).
|
||||
func (p *process) watch(ctx context.Context, cur *launched) error {
|
||||
t := time.NewTicker(healthInterval)
|
||||
defer t.Stop()
|
||||
failures := 0
|
||||
for {
|
||||
select {
|
||||
case err := <-cur.exited:
|
||||
if err == nil {
|
||||
err = errors.New("exited")
|
||||
}
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
stopChild(cur.cmd, cur.exited)
|
||||
return nil
|
||||
case <-t.C:
|
||||
hctx, cancel := context.WithTimeout(ctx, healthTimeout)
|
||||
err := cur.client.Health(hctx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
failures = 0
|
||||
continue
|
||||
}
|
||||
failures++
|
||||
if failures >= unhealthyLimit {
|
||||
stopChild(cur.cmd, cur.exited)
|
||||
return fmt.Errorf("failed %d health checks: %w", failures, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stop ends the plugin and waits for it to exit.
|
||||
func (p *process) stop() {
|
||||
p.cancel()
|
||||
<-p.done
|
||||
p.proxy.Close()
|
||||
_ = os.RemoveAll(p.sockDir)
|
||||
}
|
||||
|
||||
// stopChild asks the child to exit, then kills it after stopGrace. exited
|
||||
// is the channel its Wait result arrives on.
|
||||
func stopChild(cmd *exec.Cmd, exited chan error) {
|
||||
if cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
_ = terminate(cmd)
|
||||
select {
|
||||
case err := <-exited:
|
||||
exited <- err
|
||||
case <-time.After(stopGrace):
|
||||
_ = cmd.Process.Kill()
|
||||
err := <-exited
|
||||
exited <- err
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// egressPolicy decides which hosts a plugin may reach: the manifest's
|
||||
// permissions.egress patterns ("api.example.com", "*.atlassian.net"), which
|
||||
// the system administrator accepted at install time.
|
||||
type egressPolicy struct {
|
||||
any bool // "*": every public host
|
||||
exact map[string]bool
|
||||
suffixes []string // ".atlassian.net"
|
||||
}
|
||||
|
||||
func newEgressPolicy(patterns []string) egressPolicy {
|
||||
p := egressPolicy{exact: map[string]bool{}}
|
||||
for _, raw := range patterns {
|
||||
pat := strings.ToLower(strings.TrimSpace(raw))
|
||||
if pat == "" {
|
||||
continue
|
||||
}
|
||||
if pat == "*" {
|
||||
p.any = true
|
||||
continue
|
||||
}
|
||||
if rest, ok := strings.CutPrefix(pat, "*."); ok {
|
||||
p.suffixes = append(p.suffixes, "."+rest)
|
||||
continue
|
||||
}
|
||||
p.exact[pat] = true
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (p egressPolicy) allows(host string) bool {
|
||||
if p.any {
|
||||
// The SSRF-safe dialer still refuses private addresses.
|
||||
return true
|
||||
}
|
||||
host = strings.ToLower(strings.TrimSuffix(host, "."))
|
||||
if p.exact[host] {
|
||||
return true
|
||||
}
|
||||
for _, s := range p.suffixes {
|
||||
if strings.HasSuffix(host, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// egressProxy is the HTTP proxy a host plugin's traffic goes through
|
||||
// (HTTP_PROXY / HTTPS_PROXY in its environment). It forwards only to hosts
|
||||
// the plugin was granted and dials through the SSRF-safe dialer, so a
|
||||
// granted name that resolves to a private address is still refused unless
|
||||
// the operator whitelisted it. A plugin can ignore the proxy variables; on
|
||||
// Linux a network namespace can make the proxy the only way out.
|
||||
type egressProxy struct {
|
||||
pluginID string
|
||||
policy egressPolicy
|
||||
ln net.Listener
|
||||
srv *http.Server
|
||||
dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
logf func(format string, args ...any)
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func startEgressProxy(pluginID string, patterns []string, logf func(string, ...any)) (*egressProxy, error) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("start egress proxy: %w", err)
|
||||
}
|
||||
p := &egressProxy{
|
||||
pluginID: pluginID, policy: newEgressPolicy(patterns), ln: ln,
|
||||
dial: utils.SSRFSafeDialContext, logf: logf,
|
||||
}
|
||||
p.srv = &http.Server{Handler: p, ReadHeaderTimeout: 30 * time.Second}
|
||||
p.wg.Add(1)
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
_ = p.srv.Serve(ln)
|
||||
}()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// URL is the proxy address for the plugin's environment.
|
||||
func (p *egressProxy) URL() string { return "http://" + p.ln.Addr().String() }
|
||||
|
||||
func (p *egressProxy) Close() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = p.srv.Shutdown(ctx)
|
||||
p.wg.Wait()
|
||||
}
|
||||
|
||||
func (p *egressProxy) deny(w http.ResponseWriter, host string) {
|
||||
p.logf("[plugin] %s: egress to %s refused (not in permissions.egress)", p.pluginID, host)
|
||||
http.Error(w, fmt.Sprintf("egress to %s is not permitted for plugin %s", host, p.pluginID), http.StatusForbidden)
|
||||
}
|
||||
|
||||
func (p *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodConnect {
|
||||
p.tunnel(w, r)
|
||||
return
|
||||
}
|
||||
if r.URL.Host == "" {
|
||||
http.Error(w, "not a proxy request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
host := r.URL.Hostname()
|
||||
if !p.policy.allows(host) {
|
||||
p.deny(w, host)
|
||||
return
|
||||
}
|
||||
out := r.Clone(r.Context())
|
||||
out.RequestURI = ""
|
||||
out.Header.Del("Proxy-Connection")
|
||||
out.Header.Del("Proxy-Authorization")
|
||||
tr := &http.Transport{DialContext: p.dial, ResponseHeaderTimeout: 2 * time.Minute}
|
||||
defer tr.CloseIdleConnections()
|
||||
resp, err := tr.RoundTrip(out)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
for k, vs := range resp.Header {
|
||||
for _, v := range vs {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// tunnel handles CONNECT, which HTTPS traffic uses: the proxy sees only the
|
||||
// host and port, which is what the policy needs.
|
||||
func (p *egressProxy) tunnel(w http.ResponseWriter, r *http.Request) {
|
||||
host, _, err := net.SplitHostPort(r.Host)
|
||||
if err != nil {
|
||||
http.Error(w, "bad CONNECT target", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !p.policy.allows(host) {
|
||||
p.deny(w, host)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
upstream, err := p.dial(ctx, "tcp", r.Host)
|
||||
cancel()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
hj, ok := w.(http.Hijacker)
|
||||
if !ok {
|
||||
_ = upstream.Close()
|
||||
http.Error(w, "hijacking unsupported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
client, buf, err := hj.Hijack()
|
||||
if err != nil {
|
||||
_ = upstream.Close()
|
||||
return
|
||||
}
|
||||
if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
|
||||
_ = upstream.Close()
|
||||
_ = client.Close()
|
||||
return
|
||||
}
|
||||
if buf.Reader.Buffered() > 0 {
|
||||
pending := make([]byte, buf.Reader.Buffered())
|
||||
_, _ = buf.Read(pending)
|
||||
_, _ = upstream.Write(pending)
|
||||
}
|
||||
go func() {
|
||||
_, _ = io.Copy(upstream, client)
|
||||
_ = upstream.Close()
|
||||
}()
|
||||
_, _ = io.Copy(client, upstream)
|
||||
_ = client.Close()
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
// Command echoplugin is a host plugin for the host package's tests. Its
|
||||
// behaviour is driven by the query it is asked and by a "version" file next
|
||||
// to it, so a test can install a package that lies about its version.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
func main() {
|
||||
version := os.Getenv(pluginapi.EnvPluginVersion)
|
||||
if b, err := os.ReadFile("version"); err == nil {
|
||||
version = strings.TrimSpace(string(b))
|
||||
}
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: os.Getenv(pluginapi.EnvPluginID), Version: version})
|
||||
p.WebSearch("echo", pluginsdk.WebSearchFunc(func(_ context.Context, _ *pluginsdk.Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
|
||||
switch in.Query {
|
||||
case "crash":
|
||||
os.Exit(3)
|
||||
case "env":
|
||||
var keys []string
|
||||
for _, kv := range os.Environ() {
|
||||
k, _, _ := strings.Cut(kv, "=")
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: strings.Join(keys, ","), URL: "env"}}}, nil
|
||||
case "pid":
|
||||
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: strconv.Itoa(os.Getpid()), URL: "pid"}}}, nil
|
||||
}
|
||||
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: in.Query, URL: "echo"}}}, nil
|
||||
}))
|
||||
if err := p.Serve(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -12,7 +12,10 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
goruntime "runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/mod/semver"
|
||||
|
|
@ -41,8 +44,29 @@ func invalid(format string, args ...any) error {
|
|||
}
|
||||
|
||||
// supportedRuntimes are the runtimes an installed package may use today.
|
||||
// Code plugins arrive with the plugin host.
|
||||
var supportedRuntimes = map[manifest.RuntimeType]bool{manifest.RuntimeDeclarative: true}
|
||||
var supportedRuntimes = map[manifest.RuntimeType]bool{
|
||||
manifest.RuntimeDeclarative: true,
|
||||
manifest.RuntimeHost: true,
|
||||
}
|
||||
|
||||
// checkHostRuntime makes sure this server can run a host plugin: a binary
|
||||
// built for its OS and architecture. Other kinds need a host image that
|
||||
// carries their interpreter.
|
||||
func checkHostRuntime(p *pkg.Package) error {
|
||||
rt := p.Manifest.Runtime
|
||||
if rt.Kind != "binary" {
|
||||
return invalid("runtime.kind %q is not supported yet; host plugins must be binaries", rt.Kind)
|
||||
}
|
||||
entry := strings.NewReplacer("{os}", goruntime.GOOS, "{arch}", goruntime.GOARCH).Replace(rt.Entry)
|
||||
if goruntime.GOOS == "windows" && path.Ext(entry) == "" {
|
||||
entry += ".exe"
|
||||
}
|
||||
if _, ok := p.ReadFile(entry); !ok {
|
||||
return invalid("the package has no build for this server (%s/%s): %s is missing",
|
||||
goruntime.GOOS, goruntime.GOARCH, entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Syncer is the node's reconciler as the installer uses it.
|
||||
type Syncer interface {
|
||||
|
|
@ -152,9 +176,14 @@ func (s *Service) open(data []byte) (*pkg.Package, error) {
|
|||
return nil, &InvalidError{Err: err}
|
||||
}
|
||||
if !supportedRuntimes[p.Manifest.Runtime.Type] {
|
||||
return nil, invalid("runtime %q is not supported yet; only declarative plugins can be installed",
|
||||
return nil, invalid("runtime %q is not supported yet; declarative and host plugins can be installed",
|
||||
p.Manifest.Runtime.Type)
|
||||
}
|
||||
if p.Manifest.Runtime.Type == manifest.RuntimeHost {
|
||||
if err := checkHostRuntime(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := p.Manifest.CheckEngines(s.hostVersion); err != nil {
|
||||
return nil, &InvalidError{Err: err}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package install
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
goruntime "runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
|
|
@ -127,3 +128,36 @@ func isInvalid(err error) bool {
|
|||
var ie *InvalidError
|
||||
return errors.As(err, &ie)
|
||||
}
|
||||
|
||||
func hostPackage(t *testing.T, kind, binPath string) []byte {
|
||||
return plugintest.Zip(t, map[string]string{
|
||||
"plugin.yaml": "schemaVersion: 1\nid: acme.search\nversion: 1.0.0\napiVersion: weknora.plugin/v1\n" +
|
||||
"name: { en-US: ACME Search }\npublisher: { id: acme }\n" +
|
||||
"runtime: { type: host, kind: " + kind + ", entry: \"bin/{os}-{arch}/search\" }\n" +
|
||||
"permissions: { egress: [api.acme.example] }\n" +
|
||||
"contributes:\n webSearch:\n - { id: search, name: ACME Search }\n",
|
||||
binPath: "binary",
|
||||
})
|
||||
}
|
||||
|
||||
func TestInstallHostPlugins(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, _, _, _ := newService(t)
|
||||
name := "search"
|
||||
if goruntime.GOOS == "windows" {
|
||||
name += ".exe"
|
||||
}
|
||||
here := "bin/" + goruntime.GOOS + "-" + goruntime.GOARCH + "/" + name
|
||||
p, err := s.Inspect(ctx, hostPackage(t, "binary", here))
|
||||
if err != nil || p.Manifest.Permissions.Egress[0] != "api.acme.example" {
|
||||
t.Fatalf("Inspect = %+v, %v", p, err)
|
||||
}
|
||||
if _, err := s.Inspect(ctx, hostPackage(t, "binary", "bin/plan9-mips/search")); !isInvalid(err) ||
|
||||
!strings.Contains(err.Error(), "no build for this server") {
|
||||
t.Fatalf("want a missing build error, got %v", err)
|
||||
}
|
||||
if _, err := s.Inspect(ctx, hostPackage(t, "python", here)); !isInvalid(err) ||
|
||||
!strings.Contains(err.Error(), "must be binaries") {
|
||||
t.Fatalf("want an unsupported kind error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,15 +57,15 @@ type Manifest struct {
|
|||
|
||||
// Builtin marks a plugin compiled into WeKnora. It cannot be set from a
|
||||
// manifest file.
|
||||
Builtin bool `json:"builtin,omitempty" yaml:"-"`
|
||||
Builtin bool `json:"builtin,omitempty" yaml:"-"`
|
||||
// Required marks a builtin a tenant may not disable (the core agent
|
||||
// tools, the default parser).
|
||||
Required bool `json:"required,omitempty" yaml:"-"`
|
||||
|
||||
Runtime Runtime `json:"runtime" yaml:"runtime"`
|
||||
Permissions Permissions `json:"permissions,omitzero" yaml:"permissions"`
|
||||
Config ConfigSchemas `json:"config,omitzero" yaml:"config"`
|
||||
Contributes Contributions `json:"contributes" yaml:"contributes"`
|
||||
Runtime Runtime `json:"runtime" yaml:"runtime"`
|
||||
Permissions Permissions `json:"permissions,omitzero" yaml:"permissions"`
|
||||
Config ConfigSchemas `json:"config,omitzero" yaml:"config"`
|
||||
Contributes Contributions `json:"contributes" yaml:"contributes"`
|
||||
}
|
||||
|
||||
// Engines constrains which WeKnora versions a plugin runs on.
|
||||
|
|
@ -84,12 +84,12 @@ type Publisher struct {
|
|||
|
||||
// Runtime says how a plugin's code runs.
|
||||
type Runtime struct {
|
||||
Type RuntimeType `json:"type" yaml:"type"`
|
||||
Type RuntimeType `json:"type" yaml:"type"`
|
||||
// Kind is the process runtime for host plugins: binary, python or node.
|
||||
Kind string `json:"kind,omitempty" yaml:"kind"`
|
||||
Kind string `json:"kind,omitempty" yaml:"kind"`
|
||||
// Entry is the command inside the package, with {os} and {arch}
|
||||
// placeholders for binaries.
|
||||
Entry string `json:"entry,omitempty" yaml:"entry"`
|
||||
Entry string `json:"entry,omitempty" yaml:"entry"`
|
||||
// Singleton allows only one live instance across the cluster (long-lived
|
||||
// connections such as an IM WebSocket).
|
||||
Singleton bool `json:"singleton,omitempty" yaml:"singleton"`
|
||||
|
|
@ -106,17 +106,17 @@ type Resources struct {
|
|||
// install time.
|
||||
type Permissions struct {
|
||||
// Egress lists host patterns the plugin may reach ("*.atlassian.net").
|
||||
Egress []string `json:"egress,omitempty" yaml:"egress"`
|
||||
Egress []string `json:"egress,omitempty" yaml:"egress"`
|
||||
// HostAPI lists WeKnora API scopes the plugin may call back with.
|
||||
HostAPI []string `json:"hostApi,omitempty" yaml:"hostApi"`
|
||||
// Events lists asynchronous events the plugin subscribes to.
|
||||
Events []string `json:"events,omitempty" yaml:"events"`
|
||||
Events []string `json:"events,omitempty" yaml:"events"`
|
||||
}
|
||||
|
||||
// ConfigSchemas points at the JSON Schemas of plugin-level configuration.
|
||||
type ConfigSchemas struct {
|
||||
System string `json:"system,omitempty" yaml:"system"`
|
||||
Tenant string `json:"tenant,omitempty" yaml:"tenant"`
|
||||
System string `json:"system,omitempty" yaml:"system"`
|
||||
Tenant string `json:"tenant,omitempty" yaml:"tenant"`
|
||||
// SystemSchema and TenantSchema are the schema files' contents as JSON,
|
||||
// filled in when a package is opened.
|
||||
SystemSchema json.RawMessage `json:"systemSchema,omitempty" yaml:"-"`
|
||||
|
|
@ -130,40 +130,40 @@ type Contributions map[Point][]Contribution
|
|||
// such as one connector.
|
||||
type Contribution struct {
|
||||
// ID is local to the plugin; QualifiedID adds the plugin ID.
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Name LocalizedText `json:"name" yaml:"name"`
|
||||
Description LocalizedText `json:"description,omitzero" yaml:"description"`
|
||||
Icon string `json:"icon,omitempty" yaml:"icon"`
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Name LocalizedText `json:"name" yaml:"name"`
|
||||
Description LocalizedText `json:"description,omitzero" yaml:"description"`
|
||||
Icon string `json:"icon,omitempty" yaml:"icon"`
|
||||
// Aliases are the short names builtins were stored under before plugins
|
||||
// existed ("feishu"). Rows keep them, so lookups must keep resolving
|
||||
// them. Only builtins may declare aliases.
|
||||
Aliases []string `json:"aliases,omitempty" yaml:"-"`
|
||||
Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities"`
|
||||
Aliases []string `json:"aliases,omitempty" yaml:"-"`
|
||||
Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities"`
|
||||
// Order sorts contributions of one point in the UI (lower first); ties
|
||||
// keep registration order.
|
||||
Order int `json:"order,omitempty" yaml:"order"`
|
||||
Order int `json:"order,omitempty" yaml:"order"`
|
||||
// InstanceSchema points at the JSON Schema of one instance's
|
||||
// configuration (one data source, one IM channel).
|
||||
InstanceSchema string `json:"instanceSchema,omitempty" yaml:"instanceSchema"`
|
||||
// Path is a file or directory inside the package: the skill directory
|
||||
// (skills) or the vendor definition JSON (modelVendors).
|
||||
Path string `json:"path,omitempty" yaml:"path"`
|
||||
Path string `json:"path,omitempty" yaml:"path"`
|
||||
// MCP describes a remote MCP server (mcpServers).
|
||||
MCP *MCPServer `json:"mcp,omitempty" yaml:"mcp"`
|
||||
MCP *MCPServer `json:"mcp,omitempty" yaml:"mcp"`
|
||||
// Extra carries point-specific metadata the generic fields do not cover.
|
||||
Extra map[string]any `json:"extra,omitempty" yaml:"extra"`
|
||||
Extra map[string]any `json:"extra,omitempty" yaml:"extra"`
|
||||
}
|
||||
|
||||
// MCPServer is a remote MCP server a plugin contributes.
|
||||
type MCPServer struct {
|
||||
URL string `json:"url" yaml:"url"`
|
||||
URL string `json:"url" yaml:"url"`
|
||||
// Transport is "sse" or "http-streamable" (the default).
|
||||
Transport string `json:"transport,omitempty" yaml:"transport"`
|
||||
// Headers are sent on every request. A value may reference the
|
||||
// workspace's plugin configuration as ${config.<key>} (how a plugin asks
|
||||
// each workspace for its own API key) or the platform's as
|
||||
// ${system.<key>}. The URL is fixed: configuration cannot redirect it.
|
||||
Headers map[string]string `json:"headers,omitempty" yaml:"headers"`
|
||||
Headers map[string]string `json:"headers,omitempty" yaml:"headers"`
|
||||
}
|
||||
|
||||
// MCP transports a plugin may declare.
|
||||
|
|
@ -238,6 +238,7 @@ func (m *Manifest) Validate() error {
|
|||
}
|
||||
}
|
||||
m.validateRuntime(add)
|
||||
m.validatePermissions(add)
|
||||
m.validateContributions(add)
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
|
@ -248,8 +249,11 @@ func (m *Manifest) validateRuntime(add func(string, ...any)) {
|
|||
if !m.Builtin {
|
||||
add("runtime.type %q is reserved for builtin plugins", RuntimeBuiltin)
|
||||
}
|
||||
case RuntimeDeclarative, RuntimeRemote, RuntimeKubernetes:
|
||||
case RuntimeDeclarative:
|
||||
case RuntimeRemote, RuntimeKubernetes:
|
||||
m.validateAPIVersion(add)
|
||||
case RuntimeHost:
|
||||
m.validateAPIVersion(add)
|
||||
switch m.Runtime.Kind {
|
||||
case "binary", "python", "node":
|
||||
default:
|
||||
|
|
@ -263,6 +267,42 @@ func (m *Manifest) validateRuntime(add func(string, ...any)) {
|
|||
}
|
||||
}
|
||||
|
||||
// ExtensionAPIVersion is the extension protocol this build speaks
|
||||
// (pluginsdk/pluginapi.APIVersion).
|
||||
const ExtensionAPIVersion = "weknora.plugin/v1"
|
||||
|
||||
func (m *Manifest) validateAPIVersion(add func(string, ...any)) {
|
||||
if m.Builtin {
|
||||
return
|
||||
}
|
||||
if m.APIVersion != ExtensionAPIVersion {
|
||||
add("apiVersion must be %q for code plugins, got %q", ExtensionAPIVersion, m.APIVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// egressPattern is a host name, optionally with a leading "*." wildcard.
|
||||
var egressPattern = regexp.MustCompile(`^(\*\.)?([a-z0-9]([a-z0-9-]*[a-z0-9])?)(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$`)
|
||||
|
||||
// EgressAnyHost in permissions.egress grants any public host (a feed reader
|
||||
// follows links anywhere). Private addresses stay unreachable either way.
|
||||
const EgressAnyHost = "*"
|
||||
|
||||
func (m *Manifest) validatePermissions(add func(string, ...any)) {
|
||||
for _, e := range m.Permissions.Egress {
|
||||
if e != EgressAnyHost && !egressPattern.MatchString(strings.ToLower(e)) {
|
||||
add(
|
||||
"permissions.egress %q must be a host name (api.example.com), a wildcard (*.example.com) or *",
|
||||
e,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// storedTypeIDLimit is how long a qualified contribution ID may be where
|
||||
// instances store it in a varchar(50) column (data_sources.type,
|
||||
// web_search_providers.provider, knowledges.channel).
|
||||
var storedTypeIDLimit = map[Point]int{PointConnectors: 50, PointWebSearch: 50}
|
||||
|
||||
func (m *Manifest) validateContributions(add func(string, ...any)) {
|
||||
if len(m.Contributes) == 0 {
|
||||
add("contributes must declare at least one contribution")
|
||||
|
|
@ -297,6 +337,10 @@ func (m *Manifest) validateContributions(add func(string, ...any)) {
|
|||
if c.Name.IsZero() {
|
||||
add("%s.name is required", where)
|
||||
}
|
||||
if limit, ok := storedTypeIDLimit[point]; ok && !m.Builtin && len(QualifiedID(m.ID, c.ID)) > limit {
|
||||
add("%s: %q is longer than the %d characters instances store their type in",
|
||||
where, QualifiedID(m.ID, c.ID), limit)
|
||||
}
|
||||
if len(c.Aliases) > 0 && !m.Builtin {
|
||||
add("%s.aliases may only be declared by builtin plugins", where)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,7 +76,8 @@ func TestValidateReportsEveryProblem(t *testing.T) {
|
|||
Version: "1.0",
|
||||
Runtime: Runtime{Type: RuntimeHost},
|
||||
Contributes: Contributions{
|
||||
"widgets": {{ID: "x", Name: Text("X", nil)}},
|
||||
"widgets": {{ID: "x", Name: Text("X", nil)}},
|
||||
PointIMChannels: {{ID: "im", Name: Text("IM", nil)}},
|
||||
PointConnectors: {
|
||||
{ID: "dup", Name: Text("A", nil), Aliases: []string{"old"}},
|
||||
{ID: "dup"},
|
||||
|
|
@ -94,8 +95,9 @@ func TestValidateReportsEveryProblem(t *testing.T) {
|
|||
"name is required",
|
||||
"runtime.kind",
|
||||
"runtime.entry",
|
||||
"apiVersion must be",
|
||||
"contributes.widgets is not a known extension point",
|
||||
"is not open to third-party plugins yet",
|
||||
"contributes.imChannels is not open to third-party plugins yet",
|
||||
`"dup" is declared twice`,
|
||||
"contributes.connectors[1].name is required",
|
||||
"aliases may only be declared by builtin plugins",
|
||||
|
|
@ -184,3 +186,35 @@ func TestContributionOmitsEmptyDescription(t *testing.T) {
|
|||
t.Fatalf("empty description should be omitted: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEgressPatterns(t *testing.T) {
|
||||
base := func(egress ...string) *Manifest {
|
||||
return &Manifest{
|
||||
SchemaVersion: SchemaVersion, ID: "acme.x", Version: "1.0.0", APIVersion: ExtensionAPIVersion,
|
||||
Name: Text("X", nil), Publisher: Publisher{ID: "acme"},
|
||||
Runtime: Runtime{Type: RuntimeHost, Kind: "binary", Entry: "bin/x"},
|
||||
Permissions: Permissions{Egress: egress},
|
||||
Contributes: Contributions{PointWebSearch: {{ID: "x", Name: Text("X", nil)}}},
|
||||
}
|
||||
}
|
||||
if err := base("api.example.com", "*.atlassian.net", "*").Validate(); err != nil {
|
||||
t.Fatalf("valid egress rejected: %v", err)
|
||||
}
|
||||
for _, bad := range []string{"**", "http://x.com", "x.com:443", "*.*.com", "10.0.0.0/8"} {
|
||||
if err := base(bad).Validate(); err == nil || !strings.Contains(err.Error(), "permissions.egress") {
|
||||
t.Errorf("egress %q: got %v", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoredTypeIDLength(t *testing.T) {
|
||||
m := &Manifest{
|
||||
SchemaVersion: SchemaVersion, ID: "acme.a-rather-long-plugin-name", Version: "1.0.0",
|
||||
APIVersion: ExtensionAPIVersion, Name: Text("X", nil), Publisher: Publisher{ID: "acme"},
|
||||
Runtime: Runtime{Type: RuntimeHost, Kind: "binary", Entry: "bin/x"},
|
||||
Contributes: Contributions{PointConnectors: {{ID: "and-an-even-longer-connector-id", Name: Text("X", nil)}}},
|
||||
}
|
||||
if err := m.Validate(); err == nil || !strings.Contains(err.Error(), "longer than the 50 characters") {
|
||||
t.Fatalf("want a length error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,9 +36,9 @@ type PointInfo struct {
|
|||
// infrastructure owned by WeKnora itself, not extension points.
|
||||
var points = []PointInfo{
|
||||
{Point: PointModelVendors, ThirdParty: true, Declarative: true},
|
||||
{Point: PointConnectors},
|
||||
{Point: PointConnectors, ThirdParty: true},
|
||||
{Point: PointIMChannels},
|
||||
{Point: PointWebSearch},
|
||||
{Point: PointWebSearch, ThirdParty: true},
|
||||
{Point: PointTools},
|
||||
{Point: PointParsers},
|
||||
{Point: PointSkills, ThirdParty: true, Declarative: true},
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ func TestOpenRejectsBadPackages(t *testing.T) {
|
|||
"config/tenant.yaml": tenantSchema,
|
||||
"skills/triage/SKILL.md": "x",
|
||||
},
|
||||
want: "not open to third-party plugins",
|
||||
want: "contributes.connectors needs code",
|
||||
},
|
||||
}
|
||||
for name, c := range cases {
|
||||
|
|
|
|||
|
|
@ -106,23 +106,29 @@ func (r *Reconciler) NodeStatuses(ctx context.Context, pluginID string) ([]NodeS
|
|||
return out, nil
|
||||
}
|
||||
|
||||
// Driver returns the driver for declarative plugins. They have no code to
|
||||
// run, so an instance is a node that loaded the plugin.
|
||||
func (r *Reconciler) Driver() driver.Driver { return declarativeDriver{r} }
|
||||
// Driver returns the status driver for plugins the reconciler loads on every
|
||||
// node (declarative and host): an instance is a node that loaded the plugin,
|
||||
// with the runtime health the node reported.
|
||||
func (r *Reconciler) Driver(rt manifest.RuntimeType) driver.Driver { return nodeDriver{r: r, rt: rt} }
|
||||
|
||||
type declarativeDriver struct{ r *Reconciler }
|
||||
|
||||
func (declarativeDriver) Type() manifest.RuntimeType { return manifest.RuntimeDeclarative }
|
||||
|
||||
// Ensure and Remove are the reconciler's job for declarative plugins.
|
||||
func (declarativeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
|
||||
func (declarativeDriver) Remove(context.Context, string, string) error { return nil }
|
||||
|
||||
func (declarativeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
|
||||
return driver.Endpoint{}, fmt.Errorf("declarative plugin %s has no endpoint", pluginID)
|
||||
type nodeDriver struct {
|
||||
r *Reconciler
|
||||
rt manifest.RuntimeType
|
||||
}
|
||||
|
||||
func (d declarativeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) {
|
||||
func (d nodeDriver) Type() manifest.RuntimeType { return d.rt }
|
||||
|
||||
// Ensure and Remove are the reconciler's job.
|
||||
func (nodeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
|
||||
func (nodeDriver) Remove(context.Context, string, string) error { return nil }
|
||||
|
||||
// Resolve is not how these plugins are reached: declarative ones have no
|
||||
// endpoint and host ones are reached through the node's host manager.
|
||||
func (d nodeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
|
||||
return driver.Endpoint{}, fmt.Errorf("%s plugin %s has no shared endpoint", d.rt, pluginID)
|
||||
}
|
||||
|
||||
func (d nodeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) {
|
||||
nodes, err := d.r.NodeStatuses(ctx, pluginID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -130,7 +136,11 @@ func (d declarativeDriver) Status(ctx context.Context, pluginID string) ([]drive
|
|||
out := make([]driver.InstanceStatus, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
state := driver.StateReady
|
||||
if n.State != StateReady {
|
||||
switch n.State {
|
||||
case StateReady:
|
||||
case StateDegraded:
|
||||
state = driver.StateDegraded
|
||||
default:
|
||||
state = driver.StateStopped
|
||||
}
|
||||
out = append(out, driver.InstanceStatus{
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/plugintest"
|
||||
"github.com/Tencent/WeKnora/internal/plugin/registry"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -45,7 +46,7 @@ func TestNodesConvergeThroughRedis(t *testing.T) {
|
|||
nodes, err := a.NodeStatuses(ctx, "acme.kit")
|
||||
return err == nil && len(nodes) == 2
|
||||
})
|
||||
instances, err := b.Driver().Status(ctx, "acme.kit")
|
||||
instances, err := b.Driver(manifest.RuntimeDeclarative).Status(ctx, "acme.kit")
|
||||
if err != nil || len(instances) != 2 || instances[0].State != "ready" {
|
||||
t.Fatalf("instances = %+v, %v", instances, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,14 @@ type Activator interface {
|
|||
Deactivate(ctx context.Context, pluginID string) error
|
||||
}
|
||||
|
||||
// InPlaceActivator swaps an upgraded plugin itself inside Activate (the host
|
||||
// starts the new process before stopping the old one), so it is not
|
||||
// deactivated first.
|
||||
type InPlaceActivator interface {
|
||||
Activator
|
||||
ActivatesInPlace()
|
||||
}
|
||||
|
||||
// Status is how a plugin fares on this node.
|
||||
type Status struct {
|
||||
Version string `json:"version"`
|
||||
|
|
@ -61,6 +69,9 @@ type Status struct {
|
|||
const (
|
||||
StateReady = "ready"
|
||||
StateFailed = "failed"
|
||||
// StateDegraded: loaded, but its process crashed or fails health checks
|
||||
// and is being restarted.
|
||||
StateDegraded = "degraded"
|
||||
)
|
||||
|
||||
// DefaultInterval is how often a node reconciles without being told to.
|
||||
|
|
@ -208,7 +219,8 @@ func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) erro
|
|||
l := &Loaded{Manifest: p.Manifest, Package: p, Dir: dir}
|
||||
var errs []error
|
||||
for _, a := range r.activators {
|
||||
if _, had := r.loaded[row.ID]; had {
|
||||
_, inPlace := a.(InPlaceActivator)
|
||||
if _, had := r.loaded[row.ID]; had && !inPlace {
|
||||
if err := a.Deactivate(ctx, row.ID); err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: deactivate previous version: %w", a.Name(), err))
|
||||
}
|
||||
|
|
@ -228,7 +240,9 @@ func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) erro
|
|||
}
|
||||
|
||||
func (r *Reconciler) unload(ctx context.Context, id string) {
|
||||
for _, a := range r.activators {
|
||||
// Reverse order: routes go before the processes they route to.
|
||||
for i := len(r.activators) - 1; i >= 0; i-- {
|
||||
a := r.activators[i]
|
||||
if err := a.Deactivate(ctx, id); err != nil {
|
||||
logger.Warnf(ctx, "[plugin] %s: deactivate %s: %v", a.Name(), id, err)
|
||||
}
|
||||
|
|
@ -286,6 +300,28 @@ func (r *Reconciler) setStatus(id string, s Status) {
|
|||
r.statusMu.Unlock()
|
||||
}
|
||||
|
||||
// ReportRuntime records a loaded plugin's runtime health (a host process that
|
||||
// crashed and is restarting, or recovered) and publishes it at once.
|
||||
func (r *Reconciler) ReportRuntime(pluginID string, healthy bool, err error) {
|
||||
r.statusMu.Lock()
|
||||
s, ok := r.status[pluginID]
|
||||
if !ok || s.State == StateFailed {
|
||||
r.statusMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.State, s.Error = StateReady, ""
|
||||
if !healthy {
|
||||
s.State = StateDegraded
|
||||
if err != nil {
|
||||
s.Error = err.Error()
|
||||
}
|
||||
}
|
||||
s.UpdatedAt = time.Now()
|
||||
r.status[pluginID] = s
|
||||
r.statusMu.Unlock()
|
||||
r.publishStatuses(context.Background())
|
||||
}
|
||||
|
||||
// Status reports how one plugin fares on this node.
|
||||
func (r *Reconciler) Status(pluginID string) (Status, bool) {
|
||||
r.statusMu.RLock()
|
||||
|
|
|
|||
|
|
@ -41,7 +41,11 @@ type DataSourceService interface {
|
|||
|
||||
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
|
||||
// This is used by the frontend "Test Connection" button before creating a data source.
|
||||
ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error
|
||||
// ValidateCredentials tests unsaved credentials (and settings, which
|
||||
// some connectors need to reach the source) against the connector.
|
||||
ValidateCredentials(
|
||||
ctx context.Context, connectorType string, credentials, settings map[string]interface{},
|
||||
) error
|
||||
|
||||
// ListAvailableResources lists resources available for sync in the external system.
|
||||
// parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children.
|
||||
|
|
|
|||
|
|
@ -134,6 +134,13 @@ type WebSearchProviderTypeInfo struct {
|
|||
ID string `json:"id"`
|
||||
// Human-readable name
|
||||
Name string `json:"name"`
|
||||
// Names localizes Name for plugin provider types, keyed by locale.
|
||||
Names map[string]string `json:"names,omitempty"`
|
||||
// Icon is an image (data: URI) for plugin provider types; builtin types
|
||||
// use the frontend's bundled logos.
|
||||
Icon string `json:"icon,omitempty"`
|
||||
// PluginID names the installed plugin providing the type.
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
// Whether the provider requires an API key
|
||||
RequiresAPIKey bool `json:"requires_api_key"`
|
||||
// Whether the provider accepts an optional API key (keyless by default, but a
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
# WeKnora plugin SDK (Go)
|
||||
|
||||
Build code plugins for WeKnora: web search providers and data source
|
||||
connectors that run as their own process. The module has no dependencies.
|
||||
|
||||
| Package | What it is |
|
||||
| --- | --- |
|
||||
| `pluginapi` | The extension protocol v1: envelope, errors, stream events, messages. `openapi.yaml` describes it for other languages. |
|
||||
| `pluginsdk` | Write a plugin: register contributions, then `Serve`. |
|
||||
| `client` | Call a plugin, as WeKnora does. |
|
||||
| `conformance`, `cmd/weknora-plugin-conformance` | Check that a plugin speaks the protocol. |
|
||||
|
||||
## A connector in 30 lines
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
type notes struct{}
|
||||
|
||||
func (notes) Validate(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
|
||||
if cfg.Credentials["token"] == nil {
|
||||
return pluginapi.InvalidConfig("token is required", map[string]string{"credentials.token": "required"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (notes) ListResources(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig, parent string) ([]pluginapi.Resource, error) {
|
||||
return []pluginapi.Resource{{ExternalID: "inbox", Name: "Inbox"}}, nil
|
||||
}
|
||||
|
||||
func (notes) Fetch(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig, in pluginapi.FetchInput, s *pluginsdk.Stream) (*pluginapi.Cursor, error) {
|
||||
if err := s.Item(pluginapi.FetchedItem{ExternalID: "n1", Title: "First note", Content: []byte("# Hello")}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &pluginapi.Cursor{State: map[string]any{"after": "n1"}}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: "acme.notes", Version: "1.0.0"})
|
||||
p.Connector("notes", notes{})
|
||||
log.Fatal(p.Serve())
|
||||
}
|
||||
```
|
||||
|
||||
What the SDK does for you:
|
||||
|
||||
- **Handshake.** Started by WeKnora's plugin host, the plugin listens on the
|
||||
socket it is given and accepts only the host's token.
|
||||
- **Remote mode.** Run on its own, it listens on `WEKNORA_PLUGIN_ADDR` and
|
||||
verifies signed requests with `WEKNORA_PLUGIN_SECRET`.
|
||||
- **Per call.** Each call's deadline reaches your context. Returned
|
||||
`pluginapi` errors keep their code; anything else, and any panic, becomes
|
||||
`internal`.
|
||||
|
||||
Every call carries the tenant and the configuration, secrets already
|
||||
decrypted, in three separate scopes:
|
||||
|
||||
- `call.Config.System`: set by the system administrator.
|
||||
- `call.Config.Tenant`: set per workspace.
|
||||
- the instance: `cfg`, or `call.DecodeInstance`.
|
||||
|
||||
Keep plugins stateless and never cache a tenant's credentials across calls.
|
||||
|
||||
## Packaging
|
||||
|
||||
A package is a zip (`.wkp`) with `plugin.yaml` at its root:
|
||||
|
||||
```yaml
|
||||
schemaVersion: 1
|
||||
id: acme.notes
|
||||
version: 1.0.0
|
||||
apiVersion: weknora.plugin/v1
|
||||
name: { en-US: ACME Notes }
|
||||
publisher: { id: acme }
|
||||
runtime: { type: host, kind: binary, entry: "bin/{os}-{arch}/notes" }
|
||||
permissions:
|
||||
egress: [api.acme.example] # hosts the plugin may reach; "*" = any public host
|
||||
contributes:
|
||||
connectors:
|
||||
- id: notes
|
||||
name: { en-US: ACME Notes }
|
||||
instanceSchema: schemas/notes.yaml # fields with x-group: settings are settings, the rest credentials
|
||||
```
|
||||
|
||||
A few rules the manifest and host enforce:
|
||||
|
||||
- **Binaries.** Build one per platform into `bin/<os>-<arch>/`; WeKnora
|
||||
runs the build for its own OS and architecture.
|
||||
- **Environment.** The process gets no environment from WeKnora beyond the
|
||||
`WEKNORA_PLUGIN_*` variables.
|
||||
- **Outbound traffic.** It goes through the host's egress proxy, which
|
||||
forwards only to the hosts in `permissions.egress` and never to private
|
||||
addresses.
|
||||
- **IDs.** Contribution IDs are qualified with the plugin ID
|
||||
(`acme.notes/notes`); for connectors and web search that must stay within
|
||||
50 characters.
|
||||
|
||||
See `examples/plugins/rss` for a complete plugin and its `package.sh`.
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
go run github.com/Tencent/WeKnora/pluginsdk/cmd/weknora-plugin-conformance -url http://localhost:8080 -secret $SECRET
|
||||
```
|
||||
|
||||
In Go tests, run `conformance.Run` against `httptest.NewServer(p.Handler())`.
|
||||
|
||||
## Installing
|
||||
|
||||
In WeKnora: **System administration → Plugin management → Install plugin**.
|
||||
Upload the `.wkp` or give its URL, review what it adds and reaches, and
|
||||
install. Each workspace then enables it under **Settings → Plugins**.
|
||||
|
|
@ -0,0 +1,259 @@
|
|||
// Package client calls a WeKnora code plugin over the extension protocol. It
|
||||
// is what WeKnora itself uses, and what the conformance suite tests plugins
|
||||
// with.
|
||||
package client
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Auth adds credentials to a request.
|
||||
type Auth interface {
|
||||
Apply(req *http.Request, body []byte) error
|
||||
}
|
||||
|
||||
// Bearer authenticates as the plugin host.
|
||||
type Bearer string
|
||||
|
||||
// Apply implements Auth.
|
||||
func (b Bearer) Apply(req *http.Request, _ []byte) error {
|
||||
req.Header.Set("Authorization", "Bearer "+string(b))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Signed authenticates to a remote plugin with its shared secret.
|
||||
type Signed []byte
|
||||
|
||||
// Apply implements Auth.
|
||||
func (s Signed) Apply(req *http.Request, body []byte) error {
|
||||
ts := time.Now().Unix()
|
||||
req.Header.Set(pluginapi.TimestampHeader, strconv.FormatInt(ts, 10))
|
||||
req.Header.Set(pluginapi.SignatureHeader, pluginapi.Sign(s, ts, body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// TransportError is a failure without a protocol answer: the plugin was
|
||||
// unreachable, answered something that is not the protocol, or broke off a
|
||||
// stream. It unwraps to a *pluginapi.Error (unavailable or internal) so
|
||||
// callers can treat every failure alike.
|
||||
type TransportError struct {
|
||||
Err *pluginapi.Error
|
||||
}
|
||||
|
||||
func (e *TransportError) Error() string { return e.Err.Error() }
|
||||
func (e *TransportError) Unwrap() error { return e.Err }
|
||||
|
||||
func transport(code pluginapi.ErrorCode, format string, args ...any) error {
|
||||
return &TransportError{Err: &pluginapi.Error{
|
||||
Code: code, Message: fmt.Sprintf(format, args...), Retryable: code == pluginapi.CodeUnavailable,
|
||||
}}
|
||||
}
|
||||
|
||||
// Client calls one plugin process or service.
|
||||
type Client struct {
|
||||
base string
|
||||
http *http.Client
|
||||
auth Auth
|
||||
}
|
||||
|
||||
// ForHandshake reaches a host plugin at the address its handshake names.
|
||||
// The returned client has no overall timeout: calls are bounded by their
|
||||
// context.
|
||||
func ForHandshake(h pluginapi.Handshake, auth Auth) *Client {
|
||||
if h.Network == "unix" {
|
||||
tr := &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "unix", h.Address)
|
||||
},
|
||||
MaxIdleConnsPerHost: 16,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
return &Client{base: "http://plugin", http: &http.Client{Transport: tr}, auth: auth}
|
||||
}
|
||||
return New("http://"+h.Address, nil, auth)
|
||||
}
|
||||
|
||||
// New reaches a plugin at an HTTP(S) base URL. A nil httpClient uses a
|
||||
// client without an overall timeout.
|
||||
func New(baseURL string, httpClient *http.Client, auth Auth) *Client {
|
||||
if httpClient == nil {
|
||||
httpClient = &http.Client{}
|
||||
}
|
||||
return &Client{base: strings.TrimSuffix(baseURL, "/"), http: httpClient, auth: auth}
|
||||
}
|
||||
|
||||
// Close releases idle connections.
|
||||
func (c *Client) Close() { c.http.CloseIdleConnections() }
|
||||
|
||||
func (c *Client) do(ctx context.Context, method, path string, body []byte, requestID string) (*http.Response, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
rd = bytes.NewReader(body)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set(pluginapi.ProtocolHeader, pluginapi.ProtocolVersion)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if requestID != "" {
|
||||
req.Header.Set(pluginapi.RequestIDHeader, requestID)
|
||||
}
|
||||
if c.auth != nil {
|
||||
if err := c.auth.Apply(req, body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, transport(pluginapi.CodeUnavailable, "%v", err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// decodeError turns a non-2xx answer into a protocol error.
|
||||
func decodeError(resp *http.Response) error {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
var body pluginapi.ErrorBody
|
||||
if json.Unmarshal(b, &body) == nil && body.Error.Code != "" {
|
||||
return &body.Error
|
||||
}
|
||||
code := pluginapi.CodeInternal
|
||||
if resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusBadGateway {
|
||||
code = pluginapi.CodeUnavailable
|
||||
}
|
||||
return transport(code, "plugin answered HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
// Get reads a GET endpoint (manifest, health) into out.
|
||||
func (c *Client) Get(ctx context.Context, path string, out any) error {
|
||||
resp, err := c.do(ctx, http.MethodGet, path, nil, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return decodeError(resp)
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
// Manifest reads GET /v1/manifest.
|
||||
func (c *Client) Manifest(ctx context.Context) (*pluginapi.Manifest, error) {
|
||||
var m pluginapi.Manifest
|
||||
return &m, c.Get(ctx, "/v1/manifest", &m)
|
||||
}
|
||||
|
||||
// Health reads GET /v1/health.
|
||||
func (c *Client) Health(ctx context.Context) error {
|
||||
var h pluginapi.Health
|
||||
if err := c.Get(ctx, "/v1/health", &h); err != nil {
|
||||
return err
|
||||
}
|
||||
if h.Status != "ok" {
|
||||
return &pluginapi.Error{Code: pluginapi.CodeUnavailable, Message: "plugin reports " + h.Status, Retryable: true}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func envelopeBody(env pluginapi.Envelope, input any) ([]byte, error) {
|
||||
if input != nil {
|
||||
raw, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
env.Input = raw
|
||||
}
|
||||
return json.Marshal(env)
|
||||
}
|
||||
|
||||
// Call posts an envelope and decodes the output into out (nil discards it).
|
||||
func (c *Client) Call(ctx context.Context, path string, env pluginapi.Envelope, input, out any) error {
|
||||
body, err := envelopeBody(env, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.do(ctx, http.MethodPost, path, body, env.Context.RequestID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return decodeError(resp)
|
||||
}
|
||||
var wrapped pluginapi.Output
|
||||
if err := json.NewDecoder(resp.Body).Decode(&wrapped); err != nil {
|
||||
return transport(pluginapi.CodeInternal, "decode plugin output: %v", err)
|
||||
}
|
||||
if out == nil || len(wrapped.Output) == 0 {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(wrapped.Output, out)
|
||||
}
|
||||
|
||||
// MaxEventBytes bounds one NDJSON line (one fetched item).
|
||||
const MaxEventBytes = 64 << 20
|
||||
|
||||
// Stream posts an envelope and hands each event to fn until the stream ends.
|
||||
// It returns the "end" event's data, the "error" event as an error, or an
|
||||
// error when the stream stops without either.
|
||||
func (c *Client) Stream(
|
||||
ctx context.Context, path string, env pluginapi.Envelope, input any,
|
||||
fn func(pluginapi.Event) error,
|
||||
) (json.RawMessage, error) {
|
||||
body, err := envelopeBody(env, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.do(ctx, http.MethodPost, path, body, env.Context.RequestID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, decodeError(resp)
|
||||
}
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
sc.Buffer(make([]byte, 0, 64<<10), MaxEventBytes)
|
||||
for sc.Scan() {
|
||||
line := bytes.TrimSpace(sc.Bytes())
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var ev pluginapi.Event
|
||||
if err := json.Unmarshal(line, &ev); err != nil {
|
||||
return nil, transport(pluginapi.CodeInternal, "decode stream event: %v", err)
|
||||
}
|
||||
switch ev.Type {
|
||||
case pluginapi.EventEnd:
|
||||
return ev.Data, nil
|
||||
case pluginapi.EventError:
|
||||
if ev.Error == nil {
|
||||
return nil, &pluginapi.Error{Code: pluginapi.CodeInternal, Message: "plugin stream failed"}
|
||||
}
|
||||
return nil, ev.Error
|
||||
}
|
||||
if err := fn(ev); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, transport(pluginapi.CodeUnavailable, "stream broken: %v", err)
|
||||
}
|
||||
return nil, transport(pluginapi.CodeUnavailable, "stream ended without an end event")
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
// Command weknora-plugin-conformance checks a running plugin against the
|
||||
// extension protocol.
|
||||
//
|
||||
// weknora-plugin-conformance -url https://plugin.example.com -secret $SECRET
|
||||
// weknora-plugin-conformance -socket /tmp/p.sock -token $TOKEN
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/conformance"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
func main() {
|
||||
url := flag.String("url", "", "base URL of a remote plugin")
|
||||
socket := flag.String("socket", "", "unix socket of a plugin started by hand in host mode")
|
||||
token := flag.String("token", "", "host token (with -socket)")
|
||||
secret := flag.String("secret", "", "shared secret (with -url)")
|
||||
asJSON := flag.Bool("json", false, "print the report as JSON")
|
||||
timeout := flag.Duration("timeout", 30*time.Second, "per-check timeout")
|
||||
flag.Parse()
|
||||
|
||||
var c, anon *client.Client
|
||||
var raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
|
||||
switch {
|
||||
case *socket != "":
|
||||
hs := pluginapi.Handshake{Protocol: pluginapi.ProtocolVersion, Network: "unix", Address: *socket}
|
||||
c = client.ForHandshake(hs, client.Bearer(*token))
|
||||
anon = client.ForHandshake(hs, nil)
|
||||
case *url != "":
|
||||
var auth client.Auth
|
||||
if *secret != "" {
|
||||
auth = client.Signed(*secret)
|
||||
anon = client.New(*url, nil, nil)
|
||||
}
|
||||
c = client.New(*url, nil, auth)
|
||||
raw = func(ctx context.Context, path string, body []byte) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, *url+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if auth != nil {
|
||||
_ = auth.Apply(req, body)
|
||||
}
|
||||
return http.DefaultClient.Do(req)
|
||||
}
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "give -url or -socket")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
rep := conformance.Run(context.Background(), conformance.Target{
|
||||
Client: c, Unauthenticated: anon, Raw: raw, Timeout: *timeout,
|
||||
})
|
||||
if *asJSON {
|
||||
_ = json.NewEncoder(os.Stdout).Encode(rep)
|
||||
} else {
|
||||
fmt.Printf("plugin %s\n", rep.Plugin)
|
||||
for _, r := range rep.Results {
|
||||
mark := "PASS"
|
||||
if !r.Passed {
|
||||
mark = "FAIL"
|
||||
}
|
||||
fmt.Printf(" %s %s", mark, r.Name)
|
||||
if r.Detail != "" {
|
||||
fmt.Printf(" — %s", r.Detail)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
if !rep.Passed() {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
// Package conformance checks that a plugin speaks the extension protocol:
|
||||
// it answers the discovery endpoints, rejects what it must reject, and
|
||||
// answers every endpoint it claims with either a well-formed output or a
|
||||
// well-formed protocol error. It does not judge the answers themselves; a
|
||||
// plugin's own tests do that.
|
||||
package conformance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Result is one check.
|
||||
type Result struct {
|
||||
Name string `json:"name"`
|
||||
Passed bool `json:"passed"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Report is every check against one plugin.
|
||||
type Report struct {
|
||||
Plugin string `json:"plugin"`
|
||||
Results []Result `json:"results"`
|
||||
}
|
||||
|
||||
// Passed reports whether every check passed.
|
||||
func (r Report) Passed() bool {
|
||||
for _, res := range r.Results {
|
||||
if !res.Passed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(r.Results) > 0
|
||||
}
|
||||
|
||||
// Target is the plugin under test.
|
||||
type Target struct {
|
||||
// Client reaches the plugin with valid credentials.
|
||||
Client *client.Client
|
||||
// Unauthenticated reaches it without credentials; nil skips the check
|
||||
// (a plugin served without authentication in a test harness).
|
||||
Unauthenticated *client.Client
|
||||
// Raw posts arbitrary bytes, for malformed-request checks; nil skips.
|
||||
Raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
|
||||
// Timeout bounds each call (default 30s).
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
var semver = regexp.MustCompile(`^\d+\.\d+\.\d+([-+].*)?$`)
|
||||
|
||||
// Run checks a plugin.
|
||||
func Run(ctx context.Context, t Target) Report {
|
||||
if t.Timeout <= 0 {
|
||||
t.Timeout = 30 * time.Second
|
||||
}
|
||||
var rep Report
|
||||
check := func(name string, fn func(ctx context.Context) error) {
|
||||
cctx, cancel := context.WithTimeout(ctx, t.Timeout)
|
||||
defer cancel()
|
||||
res := Result{Name: name, Passed: true}
|
||||
if err := fn(cctx); err != nil {
|
||||
res.Passed, res.Detail = false, err.Error()
|
||||
}
|
||||
rep.Results = append(rep.Results, res)
|
||||
}
|
||||
|
||||
var m *pluginapi.Manifest
|
||||
check("manifest", func(ctx context.Context) error {
|
||||
var err error
|
||||
if m, err = t.Client.Manifest(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
rep.Plugin = m.ID + "@" + m.Version
|
||||
switch {
|
||||
case m.ID == "":
|
||||
return fmt.Errorf("manifest has no id")
|
||||
case !semver.MatchString(m.Version):
|
||||
return fmt.Errorf("version %q is not semantic", m.Version)
|
||||
case m.APIVersion != pluginapi.APIVersion:
|
||||
return fmt.Errorf("apiVersion %q, want %q", m.APIVersion, pluginapi.APIVersion)
|
||||
case len(m.Contributes) == 0:
|
||||
return fmt.Errorf("manifest lists no contributions")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
check("health", func(ctx context.Context) error { return t.Client.Health(ctx) })
|
||||
if t.Unauthenticated != nil {
|
||||
check("rejects unauthenticated calls", func(ctx context.Context) error {
|
||||
return wantCode(t.Unauthenticated.Health(ctx), pluginapi.CodeUnauthorized)
|
||||
})
|
||||
}
|
||||
check("unknown endpoint answers not_found", func(ctx context.Context) error {
|
||||
return wantCode(
|
||||
t.Client.Call(ctx, "/v1/no-such-endpoint", pluginapi.Envelope{}, nil, nil),
|
||||
pluginapi.CodeNotFound,
|
||||
)
|
||||
})
|
||||
check("unknown contribution answers not_found", func(ctx context.Context) error {
|
||||
return wantCode(t.Client.Call(ctx, pluginapi.SearchPath("no-such-provider"), pluginapi.Envelope{},
|
||||
pluginapi.SearchInput{Query: "x"}, nil), pluginapi.CodeNotFound)
|
||||
})
|
||||
if t.Raw != nil {
|
||||
check("malformed envelope answers bad_request", func(ctx context.Context) error {
|
||||
resp, err := t.Raw(ctx, "/v1/config/validate", []byte("{not json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
return wantErrorBody(resp, pluginapi.CodeBadRequest)
|
||||
})
|
||||
}
|
||||
check("config/validate answers", func(ctx context.Context) error {
|
||||
return protocolAnswer(t.Client.Call(ctx, "/v1/config/validate", envelope(), nil, nil))
|
||||
})
|
||||
if m == nil {
|
||||
return rep
|
||||
}
|
||||
for _, id := range m.Contributes["webSearch"] {
|
||||
check("webSearch/"+id+" search answers", func(ctx context.Context) error {
|
||||
var out pluginapi.SearchOutput
|
||||
err := t.Client.Call(
|
||||
ctx,
|
||||
pluginapi.SearchPath(id),
|
||||
envelope(),
|
||||
pluginapi.SearchInput{Query: "weknora", MaxResults: 3},
|
||||
&out,
|
||||
)
|
||||
if err == nil && out.Results == nil {
|
||||
return fmt.Errorf("output has no results array")
|
||||
}
|
||||
return protocolAnswer(err)
|
||||
})
|
||||
}
|
||||
for _, id := range m.Contributes["connectors"] {
|
||||
check("connectors/"+id+" validate answers", func(ctx context.Context) error {
|
||||
return protocolAnswer(t.Client.Call(ctx, pluginapi.ConnectorValidatePath(id), envelope(), nil, nil))
|
||||
})
|
||||
check("connectors/"+id+" list-resources answers", func(ctx context.Context) error {
|
||||
var out pluginapi.ListResourcesOutput
|
||||
err := t.Client.Call(
|
||||
ctx,
|
||||
pluginapi.ConnectorListResourcesPath(id),
|
||||
envelope(),
|
||||
pluginapi.ListResourcesInput{},
|
||||
&out,
|
||||
)
|
||||
if err == nil && out.Resources == nil {
|
||||
return fmt.Errorf("output has no resources array")
|
||||
}
|
||||
return protocolAnswer(err)
|
||||
})
|
||||
check("connectors/"+id+" resolve-ancestors answers", func(ctx context.Context) error {
|
||||
var out pluginapi.ResolveAncestorsOutput
|
||||
err := t.Client.Call(ctx, pluginapi.ConnectorResolveAncestorsPath(id), envelope(),
|
||||
pluginapi.ResolveAncestorsInput{ResourceIDs: []string{}}, &out)
|
||||
if err == nil && out.Ancestors == nil {
|
||||
return fmt.Errorf("output has no ancestors array")
|
||||
}
|
||||
return protocolAnswer(err)
|
||||
})
|
||||
check("connectors/"+id+" fetch stream terminates", func(ctx context.Context) error {
|
||||
_, err := t.Client.Stream(ctx, pluginapi.ConnectorFetchPath(id), envelope(),
|
||||
pluginapi.FetchInput{Mode: pluginapi.FetchFull}, func(ev pluginapi.Event) error {
|
||||
switch ev.Type {
|
||||
case pluginapi.EventItem, pluginapi.EventCheckpoint, pluginapi.EventProgress, pluginapi.EventLog:
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unknown event type %q", ev.Type)
|
||||
})
|
||||
return protocolAnswer(err)
|
||||
})
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// envelope is a call with empty configuration: plugins must answer it with
|
||||
// an output or a protocol error, never a crash or garbage.
|
||||
func envelope() pluginapi.Envelope {
|
||||
return pluginapi.Envelope{Context: pluginapi.Context{TenantID: 1, RequestID: "conformance", Locale: "en-US"}}
|
||||
}
|
||||
|
||||
// protocolAnswer accepts success or a protocol error the plugin chose, and
|
||||
// fails transport failures and internal errors: a plugin must answer an
|
||||
// empty configuration with invalid_config, not by crashing.
|
||||
func protocolAnswer(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var te *client.TransportError
|
||||
if errors.As(err, &te) {
|
||||
return fmt.Errorf("no protocol answer: %v", te)
|
||||
}
|
||||
e, ok := pluginapi.AsError(err)
|
||||
if !ok {
|
||||
return err
|
||||
}
|
||||
if e.Code == pluginapi.CodeInternal {
|
||||
return fmt.Errorf("answered internal: %s", e.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func wantCode(err error, code pluginapi.ErrorCode) error {
|
||||
e, ok := pluginapi.AsError(err)
|
||||
if !ok {
|
||||
return fmt.Errorf("want a %s protocol error, got %v", code, err)
|
||||
}
|
||||
if e.Code != code {
|
||||
return fmt.Errorf("want %s, got %s: %s", code, e.Code, e.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func wantErrorBody(resp *http.Response, code pluginapi.ErrorCode) error {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
var body pluginapi.ErrorBody
|
||||
if err := json.Unmarshal(b, &body); err != nil {
|
||||
return fmt.Errorf("HTTP %d without a protocol error body: %s", resp.StatusCode, b)
|
||||
}
|
||||
if resp.StatusCode != code.HTTPStatus() || body.Error.Code != code {
|
||||
return fmt.Errorf("want HTTP %d %s, got HTTP %d %s", code.HTTPStatus(), code, resp.StatusCode, body.Error.Code)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
package conformance_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/conformance"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
type strictConnector struct{}
|
||||
|
||||
func (strictConnector) Validate(_ context.Context, _ *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
|
||||
if cfg.Settings["url"] == nil {
|
||||
return pluginapi.InvalidConfig("url is required", map[string]string{"settings.url": "required"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s strictConnector) ListResources(
|
||||
ctx context.Context,
|
||||
call *pluginsdk.Call,
|
||||
cfg pluginsdk.ConnectorConfig,
|
||||
_ string,
|
||||
) ([]pluginapi.Resource, error) {
|
||||
return nil, s.Validate(ctx, call, cfg)
|
||||
}
|
||||
|
||||
func (s strictConnector) Fetch(
|
||||
ctx context.Context,
|
||||
call *pluginsdk.Call,
|
||||
cfg pluginsdk.ConnectorConfig,
|
||||
_ pluginapi.FetchInput,
|
||||
_ *pluginsdk.Stream,
|
||||
) (*pluginapi.Cursor, error) {
|
||||
return nil, s.Validate(ctx, call, cfg)
|
||||
}
|
||||
|
||||
func target(t *testing.T, h http.Handler) conformance.Target {
|
||||
srv := httptest.NewServer(h)
|
||||
t.Cleanup(srv.Close)
|
||||
return conformance.Target{
|
||||
Client: client.New(srv.URL, nil, nil),
|
||||
Raw: func(ctx context.Context, path string, body []byte) (*http.Response, error) {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+path, bytes.NewReader(body))
|
||||
return http.DefaultClient.Do(req)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSDKPluginConforms(t *testing.T) {
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: "acme.strict", Version: "1.0.0"})
|
||||
p.Connector("strict", strictConnector{})
|
||||
p.WebSearch(
|
||||
"none",
|
||||
pluginsdk.WebSearchFunc(
|
||||
func(context.Context, *pluginsdk.Call, pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
|
||||
return nil, pluginapi.InvalidConfig("api key required", map[string]string{"api_key": "required"})
|
||||
},
|
||||
),
|
||||
)
|
||||
rep := conformance.Run(context.Background(), target(t, p.Handler()))
|
||||
for _, r := range rep.Results {
|
||||
if !r.Passed {
|
||||
t.Errorf("%s: %s", r.Name, r.Detail)
|
||||
}
|
||||
}
|
||||
if !rep.Passed() || rep.Plugin != "acme.strict@1.0.0" || len(rep.Results) < 10 {
|
||||
t.Fatalf("report = %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrokenPluginFails(t *testing.T) {
|
||||
p := pluginsdk.New(pluginsdk.Info{ID: "acme.broken", Version: "1.0.0"})
|
||||
p.Connector("broken", strictConnector{})
|
||||
good := p.Handler()
|
||||
// A fetch that returns plain text and a list that crashes.
|
||||
broken := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/fetch"):
|
||||
_, _ = w.Write([]byte("oops"))
|
||||
case strings.HasSuffix(r.URL.Path, "/list-resources"):
|
||||
panic("nil map")
|
||||
default:
|
||||
good.ServeHTTP(w, r)
|
||||
}
|
||||
})
|
||||
rep := conformance.Run(context.Background(), target(t, broken))
|
||||
failed := map[string]bool{}
|
||||
for _, r := range rep.Results {
|
||||
if !r.Passed {
|
||||
failed[r.Name] = true
|
||||
}
|
||||
}
|
||||
if rep.Passed() || !failed["connectors/broken fetch stream terminates"] ||
|
||||
!failed["connectors/broken list-resources answers"] {
|
||||
t.Fatalf("failed checks = %v", failed)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
package pluginsdk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// ConnectorConfig is a data source's instance configuration.
|
||||
type ConnectorConfig struct {
|
||||
Credentials map[string]any `json:"credentials"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
ResourceIDs []string `json:"resourceIds"`
|
||||
}
|
||||
|
||||
// Connector is a data source connector.
|
||||
type Connector interface {
|
||||
// Validate checks the configuration, typically by calling the source.
|
||||
// Return pluginapi.InvalidConfig to point at fields.
|
||||
Validate(ctx context.Context, call *Call, cfg ConnectorConfig) error
|
||||
// ListResources lists what can be synced under parentID ("" = top).
|
||||
ListResources(ctx context.Context, call *Call, cfg ConnectorConfig, parentID string) ([]pluginapi.Resource, error)
|
||||
// Fetch streams items changed since cursor (nil = everything) and
|
||||
// returns the cursor for the next sync. Checkpoint at page boundaries
|
||||
// so an interrupted sync resumes.
|
||||
Fetch(
|
||||
ctx context.Context,
|
||||
call *Call,
|
||||
cfg ConnectorConfig,
|
||||
in pluginapi.FetchInput,
|
||||
s *Stream,
|
||||
) (*pluginapi.Cursor, error)
|
||||
}
|
||||
|
||||
// AncestorResolver is implemented by connectors whose resource tree loads
|
||||
// lazily; see pluginapi.ResolveAncestorsInput. Others answer with nothing.
|
||||
type AncestorResolver interface {
|
||||
ResolveAncestors(ctx context.Context, call *Call, cfg ConnectorConfig, resourceIDs []string) ([]string, error)
|
||||
}
|
||||
|
||||
// Connector registers connector id (contributes.connectors[].id).
|
||||
func (p *Plugin) Connector(id string, c Connector) { p.connectors[id] = c }
|
||||
|
||||
func (p *Plugin) connector(w http.ResponseWriter, r *http.Request) (Connector, bool) {
|
||||
c, ok := p.connectors[r.PathValue("id")]
|
||||
if !ok {
|
||||
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no connector %q", r.PathValue("id")))
|
||||
}
|
||||
return c, ok
|
||||
}
|
||||
|
||||
func connectorConfig(call *Call) (ConnectorConfig, error) {
|
||||
var cfg ConnectorConfig
|
||||
if err := call.DecodeInstance(&cfg); err != nil {
|
||||
return cfg, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode connector config: %v", err)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) routeConnectors(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /v1/connectors/{id}/validate", func(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := p.connector(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p.unary(func(ctx context.Context, call *Call, _ json.RawMessage) (any, error) {
|
||||
cfg, err := connectorConfig(call)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return struct{}{}, c.Validate(ctx, call, cfg)
|
||||
})(w, r)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/connectors/{id}/list-resources", func(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := p.connector(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
|
||||
in, err := decodeInput[pluginapi.ListResourcesInput](raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := connectorConfig(call)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := c.ListResources(ctx, call, cfg, in.ParentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res == nil {
|
||||
res = []pluginapi.Resource{}
|
||||
}
|
||||
return pluginapi.ListResourcesOutput{Resources: res}, nil
|
||||
})(w, r)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/connectors/{id}/resolve-ancestors", func(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := p.connector(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
|
||||
in, err := decodeInput[pluginapi.ResolveAncestorsInput](raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := pluginapi.ResolveAncestorsOutput{Ancestors: []string{}}
|
||||
resolver, ok := c.(AncestorResolver)
|
||||
if !ok {
|
||||
return out, nil
|
||||
}
|
||||
cfg, err := connectorConfig(call)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids, err := resolver.ResolveAncestors(ctx, call, cfg, in.ResourceIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ids != nil {
|
||||
out.Ancestors = ids
|
||||
}
|
||||
return out, nil
|
||||
})(w, r)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/connectors/{id}/fetch", func(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := p.connector(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
env, err := readEnvelope(r)
|
||||
if err != nil {
|
||||
writeError(w, err)
|
||||
return
|
||||
}
|
||||
in, err := decodeInput[pluginapi.FetchInput](env.Input)
|
||||
if err != nil {
|
||||
writeError(w, err)
|
||||
return
|
||||
}
|
||||
call := &Call{Context: env.Context, Config: env.Config}
|
||||
cfg, err := connectorConfig(call)
|
||||
if err != nil {
|
||||
writeError(w, err)
|
||||
return
|
||||
}
|
||||
ctx, cancel := callContext(r.Context(), env)
|
||||
defer cancel()
|
||||
s := newStream(w)
|
||||
cursor, err := c.Fetch(ctx, call, cfg, in, s)
|
||||
if err != nil {
|
||||
s.fail(err)
|
||||
return
|
||||
}
|
||||
s.end(cursor)
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
module github.com/Tencent/WeKnora/pluginsdk
|
||||
|
||||
go 1.24
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
package pluginsdk
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The OpenAPI description and the SDK's routes must list the same endpoints,
|
||||
// so a protocol change cannot land in one and not the other.
|
||||
func TestOpenAPIMatchesRoutes(t *testing.T) {
|
||||
raw, err := os.ReadFile("pluginapi/openapi.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Windows checkouts may carry CRLF line endings.
|
||||
spec := strings.ReplaceAll(string(raw), "\r\n", "\n")
|
||||
var documented []string
|
||||
for _, m := range regexp.MustCompile(`(?m)^ (/v1/[^:]+):$`).FindAllStringSubmatch(spec, -1) {
|
||||
documented = append(documented, m[1])
|
||||
}
|
||||
src, err := os.ReadFile("plugin.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range []string{"websearch.go", "connector.go"} {
|
||||
b, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src = append(src, b...)
|
||||
}
|
||||
var routed []string
|
||||
route := regexp.MustCompile(`HandleFunc\(\s*"(?:GET|POST) (/v1/[^"]+)"`)
|
||||
for _, m := range route.FindAllStringSubmatch(string(src), -1) {
|
||||
routed = append(routed, m[1])
|
||||
}
|
||||
sort.Strings(documented)
|
||||
sort.Strings(routed)
|
||||
if strings.Join(documented, "\n") != strings.Join(routed, "\n") {
|
||||
t.Fatalf(
|
||||
"openapi.yaml documents:\n%s\n\nthe SDK routes:\n%s",
|
||||
strings.Join(documented, "\n"),
|
||||
strings.Join(routed, "\n"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,349 @@
|
|||
// Package pluginsdk builds WeKnora code plugins in Go. A plugin registers what
|
||||
// it contributes and calls Serve; the SDK speaks the extension protocol
|
||||
// (package pluginapi): the handshake with the plugin host, authentication,
|
||||
// routing, streaming and error encoding.
|
||||
//
|
||||
// func main() {
|
||||
// p := pluginsdk.New(pluginsdk.Info{ID: "acme.rss", Version: "1.0.0"})
|
||||
// p.Connector("rss", &rssConnector{})
|
||||
// if err := p.Serve(); err != nil {
|
||||
// log.Fatal(err)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Started by the WeKnora plugin host, the plugin listens where the host says
|
||||
// and accepts only the host's token. Run on its own (a remote plugin), it
|
||||
// listens on WEKNORA_PLUGIN_ADDR and verifies request signatures with
|
||||
// WEKNORA_PLUGIN_SECRET.
|
||||
package pluginsdk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Info identifies the plugin; it must match plugin.yaml.
|
||||
type Info struct {
|
||||
ID string
|
||||
Version string
|
||||
}
|
||||
|
||||
// Call is one request's context and configuration. Configuration secrets are
|
||||
// already decrypted; do not keep them beyond the call.
|
||||
type Call struct {
|
||||
pluginapi.Context
|
||||
Config pluginapi.Config
|
||||
}
|
||||
|
||||
// DecodeInstance decodes the instance configuration into v.
|
||||
func (c *Call) DecodeInstance(v any) error {
|
||||
return remarshal(c.Config.Instance, v)
|
||||
}
|
||||
|
||||
// ConfigValidator checks the plugin's system and tenant configuration.
|
||||
type ConfigValidator func(ctx context.Context, call *Call) error
|
||||
|
||||
// Plugin is a plugin under construction; register contributions, then Serve.
|
||||
type Plugin struct {
|
||||
info Info
|
||||
webSearch map[string]WebSearcher
|
||||
connectors map[string]Connector
|
||||
validate ConfigValidator
|
||||
logger *slog.Logger
|
||||
// ShutdownTimeout bounds how long Serve waits for calls in flight after
|
||||
// a termination signal.
|
||||
ShutdownTimeout time.Duration
|
||||
}
|
||||
|
||||
// New starts a plugin.
|
||||
func New(info Info) *Plugin {
|
||||
return &Plugin{
|
||||
info: info,
|
||||
webSearch: map[string]WebSearcher{},
|
||||
connectors: map[string]Connector{},
|
||||
logger: slog.New(slog.NewTextHandler(os.Stderr, nil)),
|
||||
ShutdownTimeout: 60 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Logger writes to stderr, which the plugin host forwards to WeKnora's logs.
|
||||
func (p *Plugin) Logger() *slog.Logger { return p.logger }
|
||||
|
||||
// ValidateConfig registers the check behind POST /v1/config/validate.
|
||||
func (p *Plugin) ValidateConfig(fn ConfigValidator) { p.validate = fn }
|
||||
|
||||
// Manifest is what the plugin reports at GET /v1/manifest.
|
||||
func (p *Plugin) Manifest() pluginapi.Manifest {
|
||||
m := pluginapi.Manifest{
|
||||
ID: p.info.ID, Version: p.info.Version, APIVersion: pluginapi.APIVersion,
|
||||
Contributes: map[string][]string{},
|
||||
}
|
||||
add := func(point string, ids []string) {
|
||||
if len(ids) > 0 {
|
||||
sort.Strings(ids)
|
||||
m.Contributes[point] = ids
|
||||
}
|
||||
}
|
||||
add("webSearch", keys(p.webSearch))
|
||||
add("connectors", keys(p.connectors))
|
||||
return m
|
||||
}
|
||||
|
||||
func keys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Handler serves the protocol. Serve wraps it with authentication; tests
|
||||
// and custom servers can use it directly.
|
||||
func (p *Plugin) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /v1/manifest", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, p.Manifest())
|
||||
})
|
||||
mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, pluginapi.Health{Status: "ok"})
|
||||
})
|
||||
mux.HandleFunc(
|
||||
"POST /v1/config/validate",
|
||||
p.unary(func(ctx context.Context, call *Call, _ json.RawMessage) (any, error) {
|
||||
if p.validate != nil {
|
||||
if err := p.validate(ctx, call); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return struct{}{}, nil
|
||||
}),
|
||||
)
|
||||
p.routeWebSearch(mux)
|
||||
p.routeConnectors(mux)
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no endpoint %s %s", r.Method, r.URL.Path))
|
||||
})
|
||||
return recoverer(p.logger, mux)
|
||||
}
|
||||
|
||||
// unaryFunc answers one call with one output.
|
||||
type unaryFunc func(ctx context.Context, call *Call, input json.RawMessage) (any, error)
|
||||
|
||||
func (p *Plugin) unary(fn unaryFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
env, err := readEnvelope(r)
|
||||
if err != nil {
|
||||
writeError(w, err)
|
||||
return
|
||||
}
|
||||
ctx, cancel := callContext(r.Context(), env)
|
||||
defer cancel()
|
||||
out, err := fn(ctx, &Call{Context: env.Context, Config: env.Config}, env.Input)
|
||||
if err != nil {
|
||||
writeError(w, err)
|
||||
return
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
writeError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, pluginapi.Output{Output: raw})
|
||||
}
|
||||
}
|
||||
|
||||
func readEnvelope(r *http.Request) (*pluginapi.Envelope, error) {
|
||||
var env pluginapi.Envelope
|
||||
if err := json.NewDecoder(r.Body).Decode(&env); err != nil {
|
||||
return nil, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode envelope: %v", err)
|
||||
}
|
||||
return &env, nil
|
||||
}
|
||||
|
||||
// callContext applies the envelope deadline.
|
||||
func callContext(parent context.Context, env *pluginapi.Envelope) (context.Context, context.CancelFunc) {
|
||||
if env.Context.Deadline != nil {
|
||||
return context.WithDeadline(parent, *env.Context.Deadline)
|
||||
}
|
||||
return context.WithCancel(parent)
|
||||
}
|
||||
|
||||
func decodeInput[T any](raw json.RawMessage) (T, error) {
|
||||
var v T
|
||||
if len(raw) == 0 {
|
||||
return v, nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return v, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode input: %v", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func remarshal(from, to any) error {
|
||||
b, err := json.Marshal(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(b, to)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// toError maps any error to a protocol error.
|
||||
func toError(err error) *pluginapi.Error {
|
||||
if e, ok := pluginapi.AsError(err); ok {
|
||||
return e
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return pluginapi.Errorf(pluginapi.CodeUnavailable, "deadline exceeded")
|
||||
}
|
||||
return &pluginapi.Error{Code: pluginapi.CodeInternal, Message: err.Error()}
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, err error) {
|
||||
e := toError(err)
|
||||
writeJSON(w, e.Code.HTTPStatus(), pluginapi.ErrorBody{Error: *e})
|
||||
}
|
||||
|
||||
func recoverer(logger *slog.Logger, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if v := recover(); v != nil {
|
||||
logger.Error("plugin panicked", "path", r.URL.Path, "panic", v)
|
||||
writeError(w, fmt.Errorf("plugin panicked: %v", v))
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// Serve runs the plugin until SIGTERM or SIGINT, then drains calls in
|
||||
// flight. Started by the plugin host it listens where the host says and
|
||||
// prints the handshake; otherwise it serves as a remote plugin.
|
||||
func (p *Plugin) Serve() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
return p.ServeContext(ctx)
|
||||
}
|
||||
|
||||
// ServeContext is Serve with a caller-controlled lifetime.
|
||||
func (p *Plugin) ServeContext(ctx context.Context) error {
|
||||
ln, auth, handshake, err := p.listen()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srv := &http.Server{Handler: auth(p.Handler()), ReadHeaderTimeout: 10 * time.Second}
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- srv.Serve(ln) }()
|
||||
if handshake != nil {
|
||||
// The host waits for this exact line; everything else on stdout is
|
||||
// treated as log output.
|
||||
_, _ = fmt.Fprintln(os.Stdout, handshake.String())
|
||||
} else {
|
||||
p.logger.Info("plugin serving", "addr", ln.Addr().String())
|
||||
}
|
||||
select {
|
||||
case err := <-errc:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
}
|
||||
shutdown, cancel := context.WithTimeout(context.Background(), p.ShutdownTimeout)
|
||||
defer cancel()
|
||||
return srv.Shutdown(shutdown)
|
||||
}
|
||||
|
||||
type middleware func(http.Handler) http.Handler
|
||||
|
||||
// listen picks host mode (socket + token from the environment) or remote
|
||||
// mode (address + signing secret).
|
||||
func (p *Plugin) listen() (net.Listener, middleware, *pluginapi.Handshake, error) {
|
||||
if addr := os.Getenv(pluginapi.EnvSocket); addr != "" {
|
||||
network := os.Getenv(pluginapi.EnvNetwork)
|
||||
if network == "" {
|
||||
network = "unix"
|
||||
}
|
||||
token := os.Getenv(pluginapi.EnvToken)
|
||||
if token == "" {
|
||||
return nil, nil, nil, fmt.Errorf("%s is set without %s", pluginapi.EnvSocket, pluginapi.EnvToken)
|
||||
}
|
||||
if network == "unix" {
|
||||
_ = os.Remove(addr)
|
||||
}
|
||||
ln, err := net.Listen(network, addr)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("listen %s %s: %w", network, addr, err)
|
||||
}
|
||||
hs := &pluginapi.Handshake{Protocol: pluginapi.ProtocolVersion, Network: network, Address: ln.Addr().String()}
|
||||
return ln, bearerAuth(token), hs, nil
|
||||
}
|
||||
addr := os.Getenv(pluginapi.EnvAddr)
|
||||
if addr == "" {
|
||||
addr = ":8080"
|
||||
}
|
||||
secret := os.Getenv(pluginapi.EnvSecret)
|
||||
if secret == "" {
|
||||
return nil, nil, nil, fmt.Errorf(
|
||||
"a remote plugin needs %s (the secret shown when it was registered)",
|
||||
pluginapi.EnvSecret,
|
||||
)
|
||||
}
|
||||
ln, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return ln, signatureAuth([]byte(secret)), nil, nil
|
||||
}
|
||||
|
||||
func bearerAuth(token string) middleware {
|
||||
want := "Bearer " + token
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != want {
|
||||
writeError(w, pluginapi.Errorf(pluginapi.CodeUnauthorized, "missing or wrong host token"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// maxBody bounds a signed request body read into memory for verification.
|
||||
const maxBody = 32 << 20
|
||||
|
||||
func signatureAuth(secret []byte) middleware {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, maxBody+1))
|
||||
if err != nil || len(body) > maxBody {
|
||||
writeError(w, pluginapi.Errorf(pluginapi.CodeBadRequest, "request body unreadable or too large"))
|
||||
return
|
||||
}
|
||||
err = pluginapi.VerifySignature(secret, r.Header.Get(pluginapi.TimestampHeader),
|
||||
r.Header.Get(pluginapi.SignatureHeader), body, time.Now())
|
||||
if err != nil {
|
||||
writeError(w, pluginapi.Errorf(pluginapi.CodeUnauthorized, "%v", err))
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(strings.NewReader(string(body)))
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,259 @@
|
|||
package pluginsdk
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/client"
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
type feedConnector struct{ items int }
|
||||
|
||||
func (f *feedConnector) Validate(_ context.Context, _ *Call, cfg ConnectorConfig) error {
|
||||
if cfg.Settings["url"] == nil {
|
||||
return pluginapi.InvalidConfig("feed URL is required", map[string]string{"settings.url": "required"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *feedConnector) ListResources(
|
||||
_ context.Context,
|
||||
_ *Call,
|
||||
cfg ConnectorConfig,
|
||||
parentID string,
|
||||
) ([]pluginapi.Resource, error) {
|
||||
if parentID != "" {
|
||||
return nil, nil
|
||||
}
|
||||
return []pluginapi.Resource{{ExternalID: fmt.Sprint(cfg.Settings["url"]), Name: "Feed"}}, nil
|
||||
}
|
||||
|
||||
func (f *feedConnector) Fetch(
|
||||
_ context.Context, _ *Call, _ ConnectorConfig, in pluginapi.FetchInput, s *Stream,
|
||||
) (*pluginapi.Cursor, error) {
|
||||
start := 0
|
||||
if in.Cursor != nil {
|
||||
start = int(in.Cursor.State["next"].(float64))
|
||||
}
|
||||
for i := start; i < f.items; i++ {
|
||||
item := pluginapi.FetchedItem{ExternalID: fmt.Sprint(i), Title: fmt.Sprint("item ", i), Content: []byte("body")}
|
||||
if err := s.Item(item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.Checkpoint(pluginapi.Cursor{State: map[string]any{"next": i + 1}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i == 2 && in.Mode == "fail-midway" {
|
||||
return nil, pluginapi.Errorf(pluginapi.CodeUnavailable, "upstream went away")
|
||||
}
|
||||
}
|
||||
return &pluginapi.Cursor{State: map[string]any{"next": f.items}}, nil
|
||||
}
|
||||
|
||||
func testPlugin(t *testing.T) (*Plugin, *client.Client) {
|
||||
t.Helper()
|
||||
p := New(Info{ID: "acme.feed", Version: "1.0.0"})
|
||||
p.Connector("feed", &feedConnector{items: 5})
|
||||
p.WebSearch(
|
||||
"echo",
|
||||
WebSearchFunc(func(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
|
||||
switch in.Query {
|
||||
case "panic":
|
||||
panic("boom")
|
||||
case "deadline":
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{
|
||||
Title: in.Query,
|
||||
URL: fmt.Sprint("https://example.com/?tenant=", call.TenantID,
|
||||
"&key=", call.Config.Instance["api_key"]),
|
||||
}}}, nil
|
||||
}),
|
||||
)
|
||||
srv := httptest.NewServer(p.Handler())
|
||||
t.Cleanup(srv.Close)
|
||||
return p, client.New(srv.URL, nil, nil)
|
||||
}
|
||||
|
||||
func TestUnaryCallsAndErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, c := testPlugin(t)
|
||||
|
||||
m, err := c.Manifest(ctx)
|
||||
if err != nil || m.ID != "acme.feed" || m.Contributes["connectors"][0] != "feed" ||
|
||||
m.APIVersion != pluginapi.APIVersion {
|
||||
t.Fatalf("manifest = %+v, %v", m, err)
|
||||
}
|
||||
if err := c.Health(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
env := pluginapi.Envelope{
|
||||
Context: pluginapi.Context{TenantID: 7},
|
||||
Config: pluginapi.Config{Instance: map[string]any{"api_key": "k1"}},
|
||||
}
|
||||
var out pluginapi.SearchOutput
|
||||
if err := c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "hi"}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(out.Results) != 1 || out.Results[0].URL != "https://example.com/?tenant=7&key=k1" {
|
||||
t.Fatalf("search = %+v", out)
|
||||
}
|
||||
|
||||
err = c.Call(ctx, pluginapi.ConnectorValidatePath("feed"), pluginapi.Envelope{}, nil, nil)
|
||||
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeInvalidConfig ||
|
||||
e.Details.Fields["settings.url"] != "required" {
|
||||
t.Fatalf("validate = %v", err)
|
||||
}
|
||||
err = c.Call(ctx, pluginapi.SearchPath("nope"), env, pluginapi.SearchInput{}, nil)
|
||||
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeNotFound {
|
||||
t.Fatalf("unknown provider = %v", err)
|
||||
}
|
||||
err = c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "panic"}, nil)
|
||||
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeInternal ||
|
||||
!strings.Contains(e.Message, "boom") {
|
||||
t.Fatalf("panic = %v", err)
|
||||
}
|
||||
past := time.Now().Add(-time.Second)
|
||||
env.Context.Deadline = &past
|
||||
err = c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "deadline"}, nil)
|
||||
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeUnavailable {
|
||||
t.Fatalf("the envelope deadline must reach the handler's context, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingFetch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, c := testPlugin(t)
|
||||
env := pluginapi.Envelope{
|
||||
Config: pluginapi.Config{Instance: map[string]any{"settings": map[string]any{"url": "u"}}},
|
||||
}
|
||||
|
||||
var items []string
|
||||
var last *pluginapi.Cursor
|
||||
end, err := c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env,
|
||||
pluginapi.FetchInput{Mode: pluginapi.FetchFull, Cursor: &pluginapi.Cursor{State: map[string]any{"next": 3}}},
|
||||
func(ev pluginapi.Event) error {
|
||||
switch ev.Type {
|
||||
case pluginapi.EventItem:
|
||||
var it pluginapi.FetchedItem
|
||||
if err := remarshal(ev.Data, &it); err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, it.ExternalID+":"+string(it.Content))
|
||||
case pluginapi.EventCheckpoint:
|
||||
last = &pluginapi.Cursor{}
|
||||
return remarshal(ev.Data, last)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(items, ",") != "3:body,4:body" || last.State["next"].(float64) != 5 ||
|
||||
!strings.Contains(string(end), `"next":5`) {
|
||||
t.Fatalf("items=%v last=%+v end=%s", items, last, end)
|
||||
}
|
||||
|
||||
_, err = c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env, pluginapi.FetchInput{Mode: "fail-midway"},
|
||||
func(pluginapi.Event) error { return nil })
|
||||
if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeUnavailable || !e.Retryable {
|
||||
t.Fatalf("mid-stream failure = %v", err)
|
||||
}
|
||||
|
||||
stop := errors.New("ingest failed")
|
||||
_, err = c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env, pluginapi.FetchInput{},
|
||||
func(pluginapi.Event) error { return stop })
|
||||
if !errors.Is(err, stop) {
|
||||
t.Fatalf("a handler error must abort the stream, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthModes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
p, _ := testPlugin(t)
|
||||
|
||||
bearer := httptest.NewServer(bearerAuth("tok")(p.Handler()))
|
||||
defer bearer.Close()
|
||||
if err := client.New(bearer.URL, nil, client.Bearer("wrong")).Health(ctx); !isCode(
|
||||
err,
|
||||
pluginapi.CodeUnauthorized,
|
||||
) {
|
||||
t.Fatalf("wrong token = %v", err)
|
||||
}
|
||||
if err := client.New(bearer.URL, nil, client.Bearer("tok")).Health(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
signed := httptest.NewServer(signatureAuth([]byte("s3cret"))(p.Handler()))
|
||||
defer signed.Close()
|
||||
if err := client.New(signed.URL, nil, client.Signed("other")).Health(ctx); !isCode(
|
||||
err,
|
||||
pluginapi.CodeUnauthorized,
|
||||
) {
|
||||
t.Fatalf("wrong secret = %v", err)
|
||||
}
|
||||
var out pluginapi.SearchOutput
|
||||
err := client.New(signed.URL, nil, client.Signed("s3cret")).
|
||||
Call(ctx, pluginapi.SearchPath("echo"), pluginapi.Envelope{}, pluginapi.SearchInput{Query: "q"}, &out)
|
||||
if err != nil || len(out.Results) != 1 {
|
||||
t.Fatalf("signed call = %+v, %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func isCode(err error, code pluginapi.ErrorCode) bool {
|
||||
e, ok := pluginapi.AsError(err)
|
||||
return ok && e.Code == code
|
||||
}
|
||||
|
||||
// ServeContext in host mode listens on the socket from the environment,
|
||||
// prints the handshake and requires the token.
|
||||
func TestServeHostHandshake(t *testing.T) {
|
||||
dir, err := os.MkdirTemp("", "wkp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(dir) }()
|
||||
sock := filepath.Join(dir, "p.sock")
|
||||
t.Setenv(pluginapi.EnvSocket, sock)
|
||||
t.Setenv(pluginapi.EnvToken, "tok")
|
||||
|
||||
r, w, _ := os.Pipe()
|
||||
stdout := os.Stdout
|
||||
os.Stdout = w
|
||||
defer func() { os.Stdout = stdout }()
|
||||
|
||||
p, _ := testPlugin(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- p.ServeContext(ctx) }()
|
||||
|
||||
line, err := bufio.NewReader(r).ReadString('\n')
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Stdout = stdout
|
||||
hs, ok, err := pluginapi.ParseHandshake(line)
|
||||
if !ok || err != nil || hs.Network != "unix" || hs.Address != sock {
|
||||
t.Fatalf("handshake %q = %+v %v %v", line, hs, ok, err)
|
||||
}
|
||||
c := client.ForHandshake(hs, client.Bearer("tok"))
|
||||
if err := c.Health(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
package pluginapi
|
||||
|
||||
import "time"
|
||||
|
||||
// ConnectorValidatePath is the validate endpoint of connector id.
|
||||
func ConnectorValidatePath(id string) string { return "/v1/connectors/" + id + "/validate" }
|
||||
|
||||
// ConnectorListResourcesPath is the list-resources endpoint of connector id.
|
||||
func ConnectorListResourcesPath(id string) string { return "/v1/connectors/" + id + "/list-resources" }
|
||||
|
||||
// ConnectorResolveAncestorsPath is the resolve-ancestors endpoint of connector id.
|
||||
func ConnectorResolveAncestorsPath(id string) string {
|
||||
return "/v1/connectors/" + id + "/resolve-ancestors"
|
||||
}
|
||||
|
||||
// ConnectorFetchPath is the streaming fetch endpoint of connector id.
|
||||
func ConnectorFetchPath(id string) string { return "/v1/connectors/" + id + "/fetch" }
|
||||
|
||||
// The connector's instance configuration arrives in Envelope.Config.Instance
|
||||
// as {"credentials": {...}, "settings": {...}, "resourceIds": [...]}.
|
||||
|
||||
// ListResourcesInput asks for the resources under ParentID ("" = top level).
|
||||
type ListResourcesInput struct {
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
}
|
||||
|
||||
// ListResourcesOutput lists resources a user can pick for syncing.
|
||||
type ListResourcesOutput struct {
|
||||
Resources []Resource `json:"resources"`
|
||||
}
|
||||
|
||||
// Resource is something a data source can sync (a space, folder, feed).
|
||||
type Resource struct {
|
||||
ExternalID string `json:"externalId"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
ModifiedAt *time.Time `json:"modifiedAt,omitempty"`
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
HasChildren bool `json:"hasChildren,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// ResolveAncestorsInput asks which ancestors of the given resources a
|
||||
// lazily loaded picker must expand.
|
||||
type ResolveAncestorsInput struct {
|
||||
ResourceIDs []string `json:"resourceIds"`
|
||||
}
|
||||
|
||||
// ResolveAncestorsOutput is the deduplicated set of ancestor IDs.
|
||||
type ResolveAncestorsOutput struct {
|
||||
Ancestors []string `json:"ancestors"`
|
||||
}
|
||||
|
||||
// Fetch modes.
|
||||
const (
|
||||
FetchFull = "full"
|
||||
FetchIncremental = "incremental"
|
||||
)
|
||||
|
||||
// FetchInput starts or resumes a sync. The answer is a stream of "item"
|
||||
// events (FetchedItem), "checkpoint" events (Cursor) at page boundaries and
|
||||
// an "end" event carrying the final Cursor.
|
||||
type FetchInput struct {
|
||||
Mode string `json:"mode"`
|
||||
// Cursor is the last checkpoint or final cursor; nil starts from scratch.
|
||||
Cursor *Cursor `json:"cursor,omitempty"`
|
||||
// ResourceIDs are the selected resources (also in config.instance).
|
||||
ResourceIDs []string `json:"resourceIds,omitempty"`
|
||||
}
|
||||
|
||||
// Cursor is the connector's resumable state. WeKnora stores it and hands it
|
||||
// back; its content is the plugin's business. A checkpoint must be a
|
||||
// complete snapshot: resuming from it reproduces all progress so far.
|
||||
type Cursor struct {
|
||||
LastSyncTime *time.Time `json:"lastSyncTime,omitempty"`
|
||||
State map[string]any `json:"state,omitempty"`
|
||||
}
|
||||
|
||||
// FetchedItem is one document fetched from the source.
|
||||
type FetchedItem struct {
|
||||
ExternalID string `json:"externalId"`
|
||||
Title string `json:"title"`
|
||||
// Content is the document body; Markdown preferred. JSON carries it as
|
||||
// base64.
|
||||
Content []byte `json:"content,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||
CreatedAt *time.Time `json:"createdAt,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
IsDeleted bool `json:"isDeleted,omitempty"`
|
||||
SourceResourceID string `json:"sourceResourceId,omitempty"`
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
// Package pluginapi is version 1 of the WeKnora extension protocol: the
|
||||
// messages WeKnora and a code plugin exchange over HTTP. It has no
|
||||
// dependencies so that plugins and WeKnora share one definition; openapi.yaml
|
||||
// in this directory describes the same protocol for other languages.
|
||||
//
|
||||
// Every call is a POST whose body is an Envelope and whose successful answer
|
||||
// is {"output": ...}. Streaming calls answer with NDJSON Events instead. A
|
||||
// failed call answers with a non-2xx status and an ErrorBody. Within a major
|
||||
// version the protocol only grows: new optional fields and new endpoints.
|
||||
// Both sides ignore fields they do not know.
|
||||
package pluginapi
|
||||
|
||||
// ProtocolVersion is the major version this package speaks; it travels in
|
||||
// the ProtocolHeader of every request.
|
||||
const ProtocolVersion = "1"
|
||||
|
||||
// APIVersion is the manifest apiVersion of plugins speaking this protocol.
|
||||
const APIVersion = "weknora.plugin/v1"
|
||||
|
||||
// Request headers.
|
||||
const (
|
||||
ProtocolHeader = "X-WeKnora-Protocol"
|
||||
RequestIDHeader = "X-Request-Id"
|
||||
// SignatureHeader and TimestampHeader authenticate calls to remote
|
||||
// plugins: hex HMAC-SHA256 over "<timestamp>.<body>" with the shared
|
||||
// secret. Host plugins get a bearer token instead.
|
||||
SignatureHeader = "X-WeKnora-Signature"
|
||||
TimestampHeader = "X-WeKnora-Timestamp"
|
||||
)
|
||||
|
||||
// NDJSONContentType is the content type of streaming answers.
|
||||
const NDJSONContentType = "application/x-ndjson"
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package pluginapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Envelope is the body of every call. Plugins must be stateless: everything
|
||||
// about the tenant comes in the envelope, and credentials must not be cached
|
||||
// across calls.
|
||||
type Envelope struct {
|
||||
Context Context `json:"context"`
|
||||
Config Config `json:"config"`
|
||||
// Input is the endpoint-specific request, such as a SearchInput.
|
||||
Input json.RawMessage `json:"input,omitempty"`
|
||||
}
|
||||
|
||||
// Context describes who is calling and until when.
|
||||
type Context struct {
|
||||
TenantID uint64 `json:"tenantId"`
|
||||
UserID string `json:"userId,omitempty"`
|
||||
Locale string `json:"locale,omitempty"`
|
||||
RequestID string `json:"requestId,omitempty"`
|
||||
// Deadline is when WeKnora stops waiting; plugins should give up by then.
|
||||
Deadline *time.Time `json:"deadline,omitempty"`
|
||||
// Host lets the plugin call back into WeKnora (Host API); absent when
|
||||
// the plugin was granted no scopes.
|
||||
Host *HostAccess `json:"host,omitempty"`
|
||||
}
|
||||
|
||||
// HostAccess is where and how a plugin calls the WeKnora Host API.
|
||||
type HostAccess struct {
|
||||
URL string `json:"url"`
|
||||
// Token is a short-lived bearer token bound to this tenant and the
|
||||
// plugin's granted scopes.
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// Config carries the three configuration scopes separately, secrets already
|
||||
// decrypted. They are never merged, so a key means the same thing wherever
|
||||
// it appears.
|
||||
type Config struct {
|
||||
// System is the platform-wide configuration (config.system).
|
||||
System map[string]any `json:"system,omitempty"`
|
||||
// Tenant is the workspace configuration (config.tenant).
|
||||
Tenant map[string]any `json:"tenant,omitempty"`
|
||||
// Instance is the configuration of the integration instance being used:
|
||||
// a data source, a web search provider.
|
||||
Instance map[string]any `json:"instance,omitempty"`
|
||||
}
|
||||
|
||||
// Output wraps a successful answer.
|
||||
type Output struct {
|
||||
Output json.RawMessage `json:"output"`
|
||||
}
|
||||
|
||||
// Manifest is what GET /v1/manifest answers: enough for WeKnora to check that
|
||||
// the process it started is the package it installed.
|
||||
type Manifest struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
APIVersion string `json:"apiVersion"`
|
||||
// Contributes lists the contribution IDs the process serves per
|
||||
// extension point, e.g. {"connectors": ["rss"]}.
|
||||
Contributes map[string][]string `json:"contributes"`
|
||||
}
|
||||
|
||||
// Health is what GET /v1/health answers.
|
||||
type Health struct {
|
||||
Status string `json:"status"` // "ok"
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
package pluginapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ErrorCode classifies a failed call; WeKnora reacts to each differently.
|
||||
type ErrorCode string
|
||||
|
||||
const (
|
||||
// CodeInvalidConfig means the configuration is wrong; Details.Fields says
|
||||
// where. WeKnora shows it on the form.
|
||||
CodeInvalidConfig ErrorCode = "invalid_config"
|
||||
// CodeUnauthorized means the credentials no longer work.
|
||||
CodeUnauthorized ErrorCode = "unauthorized"
|
||||
// CodeNotFound means the resource does not exist.
|
||||
CodeNotFound ErrorCode = "not_found"
|
||||
// CodeRateLimited means back off for Details.RetryAfter seconds.
|
||||
CodeRateLimited ErrorCode = "rate_limited"
|
||||
// CodeUnavailable means the upstream service is down; retry later.
|
||||
CodeUnavailable ErrorCode = "unavailable"
|
||||
// CodeBadRequest means the call itself is malformed (protocol error).
|
||||
CodeBadRequest ErrorCode = "bad_request"
|
||||
// CodeInternal means the plugin failed.
|
||||
CodeInternal ErrorCode = "internal"
|
||||
)
|
||||
|
||||
// Error is a failed call. Plugins return it (or wrap it) to choose the code;
|
||||
// any other error becomes CodeInternal.
|
||||
type Error struct {
|
||||
Code ErrorCode `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Retryable bool `json:"retryable,omitempty"`
|
||||
Details *ErrorDetails `json:"details,omitempty"`
|
||||
}
|
||||
|
||||
// ErrorDetails refines some codes.
|
||||
type ErrorDetails struct {
|
||||
// Fields are per-field problems for CodeInvalidConfig, keyed by the
|
||||
// dotted field path.
|
||||
Fields map[string]string `json:"fields,omitempty"`
|
||||
// RetryAfter is seconds to wait for CodeRateLimited.
|
||||
RetryAfter int `json:"retryAfter,omitempty"`
|
||||
}
|
||||
|
||||
// ErrorBody is the JSON body of a failed call.
|
||||
type ErrorBody struct {
|
||||
Error Error `json:"error"`
|
||||
}
|
||||
|
||||
func (e *Error) Error() string { return fmt.Sprintf("%s: %s", e.Code, e.Message) }
|
||||
|
||||
// Errorf builds an Error. Unavailable and rate-limited errors are retryable.
|
||||
func Errorf(code ErrorCode, format string, args ...any) *Error {
|
||||
return &Error{
|
||||
Code: code, Message: fmt.Sprintf(format, args...),
|
||||
Retryable: code == CodeUnavailable || code == CodeRateLimited,
|
||||
}
|
||||
}
|
||||
|
||||
// InvalidConfig reports field problems, keyed by dotted path.
|
||||
func InvalidConfig(message string, fields map[string]string) *Error {
|
||||
return &Error{Code: CodeInvalidConfig, Message: message, Details: &ErrorDetails{Fields: fields}}
|
||||
}
|
||||
|
||||
// AsError finds an *Error in err's chain.
|
||||
func AsError(err error) (*Error, bool) {
|
||||
var e *Error
|
||||
ok := errors.As(err, &e)
|
||||
return e, ok
|
||||
}
|
||||
|
||||
// HTTPStatus is the status an error code travels with.
|
||||
func (c ErrorCode) HTTPStatus() int {
|
||||
switch c {
|
||||
case CodeInvalidConfig, CodeBadRequest:
|
||||
return 400
|
||||
case CodeUnauthorized:
|
||||
return 401
|
||||
case CodeNotFound:
|
||||
return 404
|
||||
case CodeRateLimited:
|
||||
return 429
|
||||
case CodeUnavailable:
|
||||
return 503
|
||||
default:
|
||||
return 500
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
package pluginapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Environment a host plugin is started with.
|
||||
const (
|
||||
// EnvSocket is where the plugin must listen (a unix socket path, or a
|
||||
// host:port when EnvNetwork is "tcp").
|
||||
EnvSocket = "WEKNORA_PLUGIN_SOCKET"
|
||||
// EnvNetwork is "unix" (default) or "tcp".
|
||||
EnvNetwork = "WEKNORA_PLUGIN_NETWORK"
|
||||
// EnvToken is the bearer token every call from the host carries.
|
||||
EnvToken = "WEKNORA_PLUGIN_TOKEN"
|
||||
// EnvPluginID and EnvPluginVersion name the package being run.
|
||||
EnvPluginID = "WEKNORA_PLUGIN_ID"
|
||||
EnvPluginVersion = "WEKNORA_PLUGIN_VERSION"
|
||||
// EnvAddr and EnvSecret configure a remote plugin (not started by a
|
||||
// host): where to listen and the shared secret calls are signed with.
|
||||
EnvAddr = "WEKNORA_PLUGIN_ADDR"
|
||||
EnvSecret = "WEKNORA_PLUGIN_SECRET"
|
||||
)
|
||||
|
||||
// handshakePrefix starts the line a host plugin prints on stdout once it is
|
||||
// ready: WEKNORA_PLUGIN|1|unix|/path/to.sock
|
||||
const handshakePrefix = "WEKNORA_PLUGIN"
|
||||
|
||||
// Handshake is the readiness line of a host plugin.
|
||||
type Handshake struct {
|
||||
Protocol string
|
||||
Network string // unix | tcp
|
||||
Address string
|
||||
}
|
||||
|
||||
// String renders the handshake line (without a newline).
|
||||
func (h Handshake) String() string {
|
||||
return strings.Join([]string{handshakePrefix, h.Protocol, h.Network, h.Address}, "|")
|
||||
}
|
||||
|
||||
// ParseHandshake reads a handshake line. ok is false for any other line,
|
||||
// which the host treats as log output.
|
||||
func ParseHandshake(line string) (h Handshake, ok bool, err error) {
|
||||
parts := strings.Split(strings.TrimSpace(line), "|")
|
||||
if len(parts) == 0 || parts[0] != handshakePrefix {
|
||||
return Handshake{}, false, nil
|
||||
}
|
||||
if len(parts) != 4 {
|
||||
return Handshake{}, true, fmt.Errorf("malformed handshake %q", line)
|
||||
}
|
||||
h = Handshake{Protocol: parts[1], Network: parts[2], Address: parts[3]}
|
||||
if h.Protocol != ProtocolVersion {
|
||||
return h, true, fmt.Errorf("plugin speaks protocol %s, host speaks %s", h.Protocol, ProtocolVersion)
|
||||
}
|
||||
if h.Network != "unix" && h.Network != "tcp" {
|
||||
return h, true, fmt.Errorf("unknown handshake network %q", h.Network)
|
||||
}
|
||||
if h.Address == "" {
|
||||
return h, true, fmt.Errorf("handshake has no address")
|
||||
}
|
||||
return h, true, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,324 @@
|
|||
openapi: 3.1.0
|
||||
info:
|
||||
title: WeKnora extension protocol
|
||||
version: "1"
|
||||
description: |
|
||||
What WeKnora calls on a code plugin. Every POST takes an Envelope and
|
||||
answers {"output": ...}, or streams NDJSON Events. Failures answer a
|
||||
non-2xx status with an ErrorBody. Within a major version the protocol
|
||||
only grows; both sides ignore fields they do not know.
|
||||
|
||||
Host plugins (started by the WeKnora plugin host) are called over the
|
||||
socket from their handshake with `Authorization: Bearer <token>`. Remote
|
||||
plugins are called over HTTPS and verify `X-WeKnora-Signature`:
|
||||
hex HMAC-SHA256 of "<X-WeKnora-Timestamp>.<body>" with the shared secret,
|
||||
rejecting timestamps more than 5 minutes off.
|
||||
servers:
|
||||
- url: http://plugin
|
||||
security:
|
||||
- hostToken: []
|
||||
- signature: []
|
||||
paths:
|
||||
/v1/manifest:
|
||||
get:
|
||||
summary: What this process is and serves
|
||||
responses:
|
||||
"200":
|
||||
description: The manifest
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Manifest" }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/health:
|
||||
get:
|
||||
summary: Liveness
|
||||
responses:
|
||||
"200":
|
||||
description: Healthy
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [status]
|
||||
properties: { status: { const: ok } }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/config/validate:
|
||||
post:
|
||||
summary: Check the plugin's system and tenant configuration
|
||||
requestBody: { $ref: "#/components/requestBodies/Envelope" }
|
||||
responses:
|
||||
"200": { $ref: "#/components/responses/EmptyOutput" }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/websearch/{id}/search:
|
||||
post:
|
||||
summary: Web search (WebSearchProvider.Search)
|
||||
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/Envelope"
|
||||
- properties: { input: { $ref: "#/components/schemas/SearchInput" } }
|
||||
responses:
|
||||
"200":
|
||||
description: Results
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [output]
|
||||
properties: { output: { $ref: "#/components/schemas/SearchOutput" } }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/connectors/{id}/validate:
|
||||
post:
|
||||
summary: Check a data source configuration (Connector.Validate)
|
||||
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
|
||||
requestBody: { $ref: "#/components/requestBodies/Envelope" }
|
||||
responses:
|
||||
"200": { $ref: "#/components/responses/EmptyOutput" }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/connectors/{id}/list-resources:
|
||||
post:
|
||||
summary: List syncable resources under a parent (Connector.ListResources)
|
||||
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/Envelope"
|
||||
- properties:
|
||||
input:
|
||||
type: object
|
||||
properties: { parentId: { type: string } }
|
||||
responses:
|
||||
"200":
|
||||
description: Resources
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [output]
|
||||
properties:
|
||||
output:
|
||||
type: object
|
||||
required: [resources]
|
||||
properties:
|
||||
resources: { type: array, items: { $ref: "#/components/schemas/Resource" } }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/connectors/{id}/resolve-ancestors:
|
||||
post:
|
||||
summary: Ancestors a lazy picker must expand to show a selection
|
||||
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/Envelope"
|
||||
- properties:
|
||||
input:
|
||||
type: object
|
||||
properties: { resourceIds: { type: array, items: { type: string } } }
|
||||
responses:
|
||||
"200":
|
||||
description: Ancestor IDs
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [output]
|
||||
properties:
|
||||
output:
|
||||
type: object
|
||||
required: [ancestors]
|
||||
properties: { ancestors: { type: array, items: { type: string } } }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
/v1/connectors/{id}/fetch:
|
||||
post:
|
||||
summary: Stream a full or incremental sync
|
||||
description: |
|
||||
Answers NDJSON: "item" events carry a FetchedItem, "checkpoint"
|
||||
events a Cursor (a complete, resumable snapshot), and the stream
|
||||
ends with "end" (data = final Cursor) or "error". A stream that
|
||||
stops without either is treated as interrupted.
|
||||
parameters: [{ $ref: "#/components/parameters/ContributionID" }]
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/Envelope"
|
||||
- properties: { input: { $ref: "#/components/schemas/FetchInput" } }
|
||||
responses:
|
||||
"200":
|
||||
description: Event stream
|
||||
content:
|
||||
application/x-ndjson:
|
||||
schema: { $ref: "#/components/schemas/Event" }
|
||||
default: { $ref: "#/components/responses/Error" }
|
||||
components:
|
||||
securitySchemes:
|
||||
hostToken: { type: http, scheme: bearer }
|
||||
signature: { type: apiKey, in: header, name: X-WeKnora-Signature }
|
||||
parameters:
|
||||
ContributionID:
|
||||
name: id
|
||||
in: path
|
||||
required: true
|
||||
description: The contribution's local ID from plugin.yaml
|
||||
schema: { type: string }
|
||||
requestBodies:
|
||||
Envelope:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/Envelope" }
|
||||
responses:
|
||||
EmptyOutput:
|
||||
description: Accepted
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [output]
|
||||
properties: { output: { type: object } }
|
||||
Error:
|
||||
description: A failed call
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: "#/components/schemas/ErrorBody" }
|
||||
schemas:
|
||||
Envelope:
|
||||
type: object
|
||||
required: [context, config]
|
||||
properties:
|
||||
context:
|
||||
type: object
|
||||
required: [tenantId]
|
||||
properties:
|
||||
tenantId: { type: integer }
|
||||
userId: { type: string }
|
||||
locale: { type: string }
|
||||
requestId: { type: string }
|
||||
deadline: { type: string, format: date-time }
|
||||
host:
|
||||
type: object
|
||||
description: Host API access; absent without granted scopes
|
||||
properties:
|
||||
url: { type: string }
|
||||
token: { type: string }
|
||||
config:
|
||||
type: object
|
||||
description: Configuration scopes, secrets decrypted, never merged
|
||||
properties:
|
||||
system: { type: object }
|
||||
tenant: { type: object }
|
||||
instance: { type: object }
|
||||
input: {}
|
||||
Manifest:
|
||||
type: object
|
||||
required: [id, version, apiVersion, contributes]
|
||||
properties:
|
||||
id: { type: string }
|
||||
version: { type: string }
|
||||
apiVersion: { const: weknora.plugin/v1 }
|
||||
contributes:
|
||||
type: object
|
||||
additionalProperties: { type: array, items: { type: string } }
|
||||
SearchInput:
|
||||
type: object
|
||||
required: [query]
|
||||
properties:
|
||||
query: { type: string }
|
||||
maxResults: { type: integer }
|
||||
includeDate: { type: boolean }
|
||||
region: { type: string }
|
||||
freshness: { type: string }
|
||||
SearchOutput:
|
||||
type: object
|
||||
required: [results]
|
||||
properties:
|
||||
results:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
required: [title, url]
|
||||
properties:
|
||||
title: { type: string }
|
||||
url: { type: string }
|
||||
snippet: { type: string }
|
||||
content: { type: string }
|
||||
age: { type: string }
|
||||
publishedAt: { type: string, format: date-time }
|
||||
Resource:
|
||||
type: object
|
||||
required: [externalId, name]
|
||||
properties:
|
||||
externalId: { type: string }
|
||||
name: { type: string }
|
||||
type: { type: string }
|
||||
description: { type: string }
|
||||
url: { type: string }
|
||||
modifiedAt: { type: string, format: date-time }
|
||||
parentId: { type: string }
|
||||
hasChildren: { type: boolean }
|
||||
metadata: { type: object }
|
||||
FetchInput:
|
||||
type: object
|
||||
required: [mode]
|
||||
properties:
|
||||
mode: { enum: [full, incremental] }
|
||||
cursor: { $ref: "#/components/schemas/Cursor" }
|
||||
resourceIds: { type: array, items: { type: string } }
|
||||
Cursor:
|
||||
type: object
|
||||
properties:
|
||||
lastSyncTime: { type: string, format: date-time }
|
||||
state: { type: object }
|
||||
FetchedItem:
|
||||
type: object
|
||||
required: [externalId, title]
|
||||
properties:
|
||||
externalId: { type: string }
|
||||
title: { type: string }
|
||||
content: { type: string, contentEncoding: base64 }
|
||||
contentType: { type: string }
|
||||
fileName: { type: string }
|
||||
url: { type: string }
|
||||
updatedAt: { type: string, format: date-time }
|
||||
createdAt: { type: string, format: date-time }
|
||||
metadata: { type: object, additionalProperties: { type: string } }
|
||||
isDeleted: { type: boolean }
|
||||
sourceResourceId: { type: string }
|
||||
Event:
|
||||
type: object
|
||||
required: [type]
|
||||
properties:
|
||||
type: { enum: [item, checkpoint, progress, log, error, end] }
|
||||
data: {}
|
||||
error: { $ref: "#/components/schemas/Error" }
|
||||
message: { type: string }
|
||||
level: { enum: [debug, info, warn, error] }
|
||||
Error:
|
||||
type: object
|
||||
required: [code, message]
|
||||
properties:
|
||||
code: { enum: [invalid_config, unauthorized, not_found, rate_limited, unavailable, bad_request, internal] }
|
||||
message: { type: string }
|
||||
retryable: { type: boolean }
|
||||
details:
|
||||
type: object
|
||||
properties:
|
||||
fields: { type: object, additionalProperties: { type: string } }
|
||||
retryAfter: { type: integer }
|
||||
ErrorBody:
|
||||
type: object
|
||||
required: [error]
|
||||
properties:
|
||||
error: { $ref: "#/components/schemas/Error" }
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package pluginapi
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHandshake(t *testing.T) {
|
||||
h := Handshake{Protocol: ProtocolVersion, Network: "unix", Address: "/tmp/p.sock"}
|
||||
got, ok, err := ParseHandshake(h.String() + "\n")
|
||||
if !ok || err != nil || got != h {
|
||||
t.Fatalf("round trip = %+v %v %v", got, ok, err)
|
||||
}
|
||||
if _, ok, _ := ParseHandshake("starting up..."); ok {
|
||||
t.Fatal("other output is not a handshake")
|
||||
}
|
||||
bad := []string{
|
||||
"WEKNORA_PLUGIN|2|unix|/x", "WEKNORA_PLUGIN|1|pipe|/x", "WEKNORA_PLUGIN|1|unix|", "WEKNORA_PLUGIN|1",
|
||||
}
|
||||
for _, bad := range bad {
|
||||
if _, ok, err := ParseHandshake(bad); !ok || err == nil {
|
||||
t.Errorf("%q must be a rejected handshake", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignature(t *testing.T) {
|
||||
secret, body, now := []byte("k"), []byte(`{"a":1}`), time.Now()
|
||||
ts := strconv.FormatInt(now.Unix(), 10)
|
||||
sig := Sign(secret, now.Unix(), body)
|
||||
if err := VerifySignature(secret, ts, sig, body, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if VerifySignature(secret, ts, sig, []byte(`{"a":2}`), now) == nil {
|
||||
t.Fatal("a changed body must fail")
|
||||
}
|
||||
if VerifySignature(secret, ts, sig, body, now.Add(10*time.Minute)) == nil {
|
||||
t.Fatal("a replayed old request must fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorStatus(t *testing.T) {
|
||||
if Errorf(CodeRateLimited, "x").Code.HTTPStatus() != 429 || !Errorf(CodeUnavailable, "x").Retryable ||
|
||||
Errorf(CodeInternal, "x").Retryable {
|
||||
t.Fatal("error codes map wrong")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package pluginapi
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MaxClockSkew bounds how old a signed request may be.
|
||||
const MaxClockSkew = 5 * time.Minute
|
||||
|
||||
// Sign returns the signature of a request body at a unix timestamp.
|
||||
func Sign(secret []byte, timestamp int64, body []byte) string {
|
||||
mac := hmac.New(sha256.New, secret)
|
||||
_, _ = fmt.Fprintf(mac, "%d.", timestamp)
|
||||
_, _ = mac.Write(body)
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// VerifySignature checks a signed request against the secret and clock.
|
||||
func VerifySignature(secret []byte, timestamp, signature string, body []byte, now time.Time) error {
|
||||
ts, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad %s", TimestampHeader)
|
||||
}
|
||||
if d := now.Sub(time.Unix(ts, 0)); d > MaxClockSkew || d < -MaxClockSkew {
|
||||
return fmt.Errorf("request timestamp is outside the allowed clock skew")
|
||||
}
|
||||
want := Sign(secret, ts, body)
|
||||
if !hmac.Equal([]byte(want), []byte(signature)) {
|
||||
return fmt.Errorf("bad signature")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package pluginapi
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// EventType is the kind of one NDJSON line in a streaming answer.
|
||||
type EventType string
|
||||
|
||||
const (
|
||||
// EventItem carries one result (a FetchedItem for connectors).
|
||||
EventItem EventType = "item"
|
||||
// EventCheckpoint carries a resumable cursor; WeKnora persists it.
|
||||
EventCheckpoint EventType = "checkpoint"
|
||||
// EventProgress reports progress for display.
|
||||
EventProgress EventType = "progress"
|
||||
// EventLog forwards a log line to WeKnora's logs.
|
||||
EventLog EventType = "log"
|
||||
// EventError ends the stream with an error.
|
||||
EventError EventType = "error"
|
||||
// EventEnd ends the stream successfully, with the final Data.
|
||||
EventEnd EventType = "end"
|
||||
)
|
||||
|
||||
// Event is one line of a streaming answer. A stream that stops without an
|
||||
// "end" or "error" event was interrupted and is treated as failed.
|
||||
type Event struct {
|
||||
Type EventType `json:"type"`
|
||||
Data json.RawMessage `json:"data,omitempty"`
|
||||
// Error is set on EventError.
|
||||
Error *Error `json:"error,omitempty"`
|
||||
// Message is set on EventLog and EventProgress.
|
||||
Message string `json:"message,omitempty"`
|
||||
// Level is set on EventLog: debug, info, warn, error.
|
||||
Level string `json:"level,omitempty"`
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package pluginapi
|
||||
|
||||
import "time"
|
||||
|
||||
// SearchPath is the web search endpoint for provider id.
|
||||
func SearchPath(id string) string { return "/v1/websearch/" + id + "/search" }
|
||||
|
||||
// SearchInput is the input of a web search.
|
||||
type SearchInput struct {
|
||||
Query string `json:"query"`
|
||||
MaxResults int `json:"maxResults"`
|
||||
IncludeDate bool `json:"includeDate,omitempty"`
|
||||
// Region and Freshness are optional filters; a provider that cannot
|
||||
// honour a requested filter must fail rather than ignore it.
|
||||
Region string `json:"region,omitempty"`
|
||||
Freshness string `json:"freshness,omitempty"`
|
||||
}
|
||||
|
||||
// SearchOutput is the output of a web search.
|
||||
type SearchOutput struct {
|
||||
Results []SearchResult `json:"results"`
|
||||
}
|
||||
|
||||
// SearchResult is one hit.
|
||||
type SearchResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Age string `json:"age,omitempty"`
|
||||
PublishedAt *time.Time `json:"publishedAt,omitempty"`
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
package pluginsdk
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// Stream writes a streaming answer: items, checkpoints and progress as NDJSON
|
||||
// lines, flushed one by one. It is safe for concurrent use.
|
||||
type Stream struct {
|
||||
mu sync.Mutex
|
||||
w http.ResponseWriter
|
||||
flusher http.Flusher
|
||||
started bool
|
||||
closed bool
|
||||
err error
|
||||
}
|
||||
|
||||
func newStream(w http.ResponseWriter) *Stream {
|
||||
f, _ := w.(http.Flusher)
|
||||
return &Stream{w: w, flusher: f}
|
||||
}
|
||||
|
||||
func (s *Stream) write(ev pluginapi.Event) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.err != nil {
|
||||
return s.err
|
||||
}
|
||||
if s.closed {
|
||||
return pluginapi.Errorf(pluginapi.CodeInternal, "stream already ended")
|
||||
}
|
||||
if !s.started {
|
||||
s.w.Header().Set("Content-Type", pluginapi.NDJSONContentType)
|
||||
s.w.WriteHeader(http.StatusOK)
|
||||
s.started = true
|
||||
}
|
||||
b, err := json.Marshal(ev)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.w.Write(append(b, '\n')); err != nil {
|
||||
// The caller went away; stop producing.
|
||||
s.err = err
|
||||
return err
|
||||
}
|
||||
if s.flusher != nil {
|
||||
s.flusher.Flush()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataEvent(t pluginapi.EventType, v any) (pluginapi.Event, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return pluginapi.Event{}, err
|
||||
}
|
||||
return pluginapi.Event{Type: t, Data: raw}, nil
|
||||
}
|
||||
|
||||
// Item emits one fetched item. An error means WeKnora stopped listening:
|
||||
// stop fetching and return.
|
||||
func (s *Stream) Item(item pluginapi.FetchedItem) error {
|
||||
ev, err := dataEvent(pluginapi.EventItem, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.write(ev)
|
||||
}
|
||||
|
||||
// Checkpoint emits a resumable cursor. It must be a complete snapshot.
|
||||
func (s *Stream) Checkpoint(c pluginapi.Cursor) error {
|
||||
ev, err := dataEvent(pluginapi.EventCheckpoint, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.write(ev)
|
||||
}
|
||||
|
||||
// Progress reports progress for display.
|
||||
func (s *Stream) Progress(message string) error {
|
||||
return s.write(pluginapi.Event{Type: pluginapi.EventProgress, Message: message})
|
||||
}
|
||||
|
||||
// Log forwards a line to WeKnora's logs (level: debug, info, warn, error).
|
||||
func (s *Stream) Log(level, message string) error {
|
||||
return s.write(pluginapi.Event{Type: pluginapi.EventLog, Level: level, Message: message})
|
||||
}
|
||||
|
||||
func (s *Stream) end(cursor *pluginapi.Cursor) {
|
||||
if cursor == nil {
|
||||
cursor = &pluginapi.Cursor{}
|
||||
}
|
||||
ev, err := dataEvent(pluginapi.EventEnd, cursor)
|
||||
if err == nil {
|
||||
err = s.write(ev)
|
||||
}
|
||||
if err != nil {
|
||||
s.fail(err)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// fail ends the stream with an error: as an error event once lines went
|
||||
// out, or as an ordinary error answer before that.
|
||||
func (s *Stream) fail(err error) {
|
||||
s.mu.Lock()
|
||||
started, closed := s.started, s.closed
|
||||
s.mu.Unlock()
|
||||
if closed {
|
||||
return
|
||||
}
|
||||
if !started {
|
||||
writeError(s.w, err)
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
_ = s.write(pluginapi.Event{Type: pluginapi.EventError, Error: toError(err)})
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
package pluginsdk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
||||
)
|
||||
|
||||
// WebSearcher is a web search provider. Its instance configuration is what
|
||||
// the contribution's instanceSchema describes.
|
||||
type WebSearcher interface {
|
||||
Search(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error)
|
||||
}
|
||||
|
||||
// WebSearchFunc adapts a function to WebSearcher.
|
||||
type WebSearchFunc func(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error)
|
||||
|
||||
// Search implements WebSearcher.
|
||||
func (f WebSearchFunc) Search(
|
||||
ctx context.Context,
|
||||
call *Call,
|
||||
in pluginapi.SearchInput,
|
||||
) (*pluginapi.SearchOutput, error) {
|
||||
return f(ctx, call, in)
|
||||
}
|
||||
|
||||
// WebSearch registers web search provider id (contributes.webSearch[].id).
|
||||
func (p *Plugin) WebSearch(id string, s WebSearcher) { p.webSearch[id] = s }
|
||||
|
||||
func (p *Plugin) routeWebSearch(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /v1/websearch/{id}/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
s, ok := p.webSearch[r.PathValue("id")]
|
||||
if !ok {
|
||||
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no web search provider %q", r.PathValue("id")))
|
||||
return
|
||||
}
|
||||
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
|
||||
in, err := decodeInput[pluginapi.SearchInput](raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := s.Search(ctx, call, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = &pluginapi.SearchOutput{}
|
||||
}
|
||||
if out.Results == nil {
|
||||
out.Results = []pluginapi.SearchResult{}
|
||||
}
|
||||
return out, nil
|
||||
})(w, r)
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue