-
{{ p.id }} · v{{ p.active_version }} · {{ p.runtime }}
+
{{ p.id }} · v{{ p.active_version }} · {{ runtimeLabel(p.runtime) }}
{{ descriptionOf(p) }}
@@ -89,7 +89,8 @@ import { installedState } from './pluginManagementState'
// Platform plugin management: what is installed, on which version, whether it
// loads. Installing makes a plugin available to every workspace; each
// workspace still turns it on in its own plugin center.
-const { t, locale } = useI18n()
+const { t, te, locale } = useI18n()
+const runtimeLabel = (rt: string) => (te(`pluginAdmin.runtime.${rt}`) ? t(`pluginAdmin.runtime.${rt}`) : rt)
const plugins = ref([])
const loading = ref(false)
@@ -109,6 +110,8 @@ function stateTheme(p: InstalledPlugin) {
return 'success'
case 'failed':
return 'danger'
+ case 'degraded':
+ return 'warning'
default:
return 'default'
}
diff --git a/frontend/src/views/system/pluginManagementState.test.ts b/frontend/src/views/system/pluginManagementState.test.ts
index 40209714..2e03b366 100644
--- a/frontend/src/views/system/pluginManagementState.test.ts
+++ b/frontend/src/views/system/pluginManagementState.test.ts
@@ -67,6 +67,7 @@ test('installedState reads desired state and the node report', () => {
assert.equal(installedState(base), 'pending')
assert.equal(installedState({ ...base, node: { version: '1', state: 'ready', updatedAt: '' } }), 'running')
assert.equal(installedState({ ...base, node: { version: '1', state: 'failed', updatedAt: '' } }), 'failed')
+ assert.equal(installedState({ ...base, node: { version: '1', state: 'degraded', updatedAt: '' } }), 'degraded')
})
test('isPackageUrl and hasSystemConfig', () => {
diff --git a/frontend/src/views/system/pluginManagementState.ts b/frontend/src/views/system/pluginManagementState.ts
index d19aa781..c1c8c448 100644
--- a/frontend/src/views/system/pluginManagementState.ts
+++ b/frontend/src/views/system/pluginManagementState.ts
@@ -95,14 +95,24 @@ export function sortVersions(list: readonly PluginVersion[]): PluginVersion[] {
}
/** Overall state of an installed plugin on the node that answered. */
-export type InstalledState = 'running' | 'failed' | 'disabled' | 'pending'
+export type InstalledState = 'running' | 'degraded' | 'failed' | 'disabled' | 'pending'
export function installedState(p: InstalledPlugin): InstalledState {
if (p.desired_state === 'disabled') return 'disabled'
if (!p.node) return 'pending'
- return p.node.state === 'ready' ? 'running' : 'failed'
+ switch (p.node.state) {
+ case 'ready':
+ return 'running'
+ case 'degraded':
+ return 'degraded'
+ default:
+ return 'failed'
+ }
}
+/** The egress grant meaning any public host. */
+export const EGRESS_ANY_HOST = '*'
+
/** Whether a string can be sent as a package URL. */
export function isPackageUrl(raw: string): boolean {
try {
diff --git a/frontend/src/views/system/plugins/PluginDetailDrawer.vue b/frontend/src/views/system/plugins/PluginDetailDrawer.vue
index d71d2e4b..1a1c8d09 100644
--- a/frontend/src/views/system/plugins/PluginDetailDrawer.vue
+++ b/frontend/src/views/system/plugins/PluginDetailDrawer.vue
@@ -19,7 +19,7 @@
{{ t('pluginAdmin.publisher') }}
{{ manifest?.publisher.name || manifest?.publisher.id }}
{{ t('pluginAdmin.detail.runtime') }}
- {{ plugin.runtime }}
+ {{ te(`pluginAdmin.runtime.${plugin.runtime}`) ? t(`pluginAdmin.runtime.${plugin.runtime}`) : plugin.runtime }}
{{ t('pluginAdmin.detail.source') }}
{{ t(`pluginAdmin.source.${plugin.source?.kind ?? 'upload'}`) }}
@@ -141,7 +141,7 @@ const emit = defineEmits<{
removed: [id: string]
}>()
-const { t, locale } = useI18n()
+const { t, te, locale } = useI18n()
const manifest = computed(() => props.plugin?.manifest)
const title = computed(() => (manifest.value ? localizedText(manifest.value.name, locale.value) : props.plugin?.id ?? ''))
diff --git a/frontend/src/views/system/plugins/PluginInstallDrawer.vue b/frontend/src/views/system/plugins/PluginInstallDrawer.vue
index e009b475..2a9058aa 100644
--- a/frontend/src/views/system/plugins/PluginInstallDrawer.vue
+++ b/frontend/src/views/system/plugins/PluginInstallDrawer.vue
@@ -96,7 +96,10 @@
{{ t(`pluginAdmin.permission.${p.kind}`) }}
- {{ p.value }}
+
+ {{ t('pluginAdmin.permission.anyHost') }}
+
+ {{ p.value }}
@@ -128,7 +131,15 @@ import {
} from '@/api/system/plugins'
import { localizedText } from '@/utils/localizedText'
-import { hasSystemConfig, contributionLines, formatBytes, isPackageUrl, permissionLines, remoteHosts } from '../pluginManagementState'
+import {
+ EGRESS_ANY_HOST,
+ contributionLines,
+ formatBytes,
+ hasSystemConfig,
+ isPackageUrl,
+ permissionLines,
+ remoteHosts,
+} from '../pluginManagementState'
import { hasTenantConfig } from '../../settings/pluginCenterState'
// Installing is two steps on purpose: inspect shows what the package would
@@ -393,6 +404,12 @@ async function onConfirm() {
color: var(--td-text-color-placeholder);
word-break: break-all;
}
+
+ &__warn {
+ font-size: var(--app-text-sm);
+ font-weight: 500;
+ color: var(--td-warning-color);
+ }
}
.review-alert {
diff --git a/go.mod b/go.mod
index 20ae0700..1eec7edc 100644
--- a/go.mod
+++ b/go.mod
@@ -100,6 +100,7 @@ require (
github.com/JohannesKaufmann/dom v0.2.0 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
+ github.com/Tencent/WeKnora/pluginsdk v0.0.0-00010101000000-000000000000
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/apache/arrow-go/v18 v18.5.1 // indirect
@@ -359,3 +360,5 @@ replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelg
// The anydoc Go bindings are not published yet (firecrawl/anydoc#30 is still
// open), so they are vendored. Drop this replace once upstream tags go/vX.Y.Z.
replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go
+
+replace github.com/Tencent/WeKnora/pluginsdk => ./pluginsdk
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index fe106dfe..cf9b49f6 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -658,7 +658,9 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
}
ds.ErrorMessage = syncLog.ErrorMessage
_ = s.dsRepo.Update(ctx, ds)
- return err
+ // Retrying cannot bring a connector back (its plugin was removed or
+ // disabled); the next scheduled or manual sync will try again.
+ return fmt.Errorf("%w: %w", asynq.SkipRetry, err)
}
// Parse configuration
@@ -1247,7 +1249,9 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {
}
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
-func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error {
+func (s *DataSourceService) ValidateCredentials(
+ ctx context.Context, connectorType string, credentials, settings map[string]interface{},
+) error {
connector, err := s.connectorRegistry.Get(connectorType)
if err != nil {
return err
@@ -1255,6 +1259,7 @@ func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorTy
config := &types.DataSourceConfig{
Type: connectorType,
Credentials: credentials,
+ Settings: settings,
}
if err := connector.Validate(ctx, config); err != nil {
return err
diff --git a/internal/application/service/web_search_provider.go b/internal/application/service/web_search_provider.go
index 336b2201..105573f2 100644
--- a/internal/application/service/web_search_provider.go
+++ b/internal/application/service/web_search_provider.go
@@ -14,11 +14,46 @@ import (
// webSearchProviderService implements interfaces.WebSearchProviderService
type webSearchProviderService struct {
repo interfaces.WebSearchProviderRepository
+ // registry knows the provider types installed plugins contribute.
+ registry *infra_web_search.Registry
}
// NewWebSearchProviderService creates a new web search provider service
-func NewWebSearchProviderService(repo interfaces.WebSearchProviderRepository) interfaces.WebSearchProviderService {
- return &webSearchProviderService{repo: repo}
+func NewWebSearchProviderService(
+ repo interfaces.WebSearchProviderRepository, registry *infra_web_search.Registry,
+) interfaces.WebSearchProviderService {
+ return &webSearchProviderService{repo: repo, registry: registry}
+}
+
+// validType accepts builtin types and the types of loaded plugins.
+func (s *webSearchProviderService) validType(t types.WebSearchProviderType) bool {
+ if isValidProviderType(t) {
+ return true
+ }
+ if s.registry == nil {
+ return false
+ }
+ _, ok := s.registry.PluginType(string(t))
+ return ok
+}
+
+// validateParams checks builtin parameters by type and plugin parameters
+// against the plugin's instance schema.
+func (s *webSearchProviderService) validateParams(
+ t types.WebSearchProviderType, params types.WebSearchProviderParameters,
+) error {
+ if s.registry != nil {
+ if pt, ok := s.registry.PluginType(string(t)); ok {
+ if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
+ return err
+ }
+ if pt.Validate != nil {
+ return pt.Validate(params)
+ }
+ return nil
+ }
+ }
+ return validateProviderParameters(t, params)
}
// CreateProvider creates a new web search provider configuration.
@@ -27,11 +62,11 @@ func (s *webSearchProviderService) CreateProvider(ctx context.Context, provider
return fmt.Errorf("tenant ID is required")
}
- if !isValidProviderType(provider.Provider) {
+ if !s.validType(provider.Provider) {
return fmt.Errorf("invalid provider type: %s", provider.Provider)
}
- if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil {
+ if err := s.validateParams(provider.Provider, provider.Parameters); err != nil {
return err
}
@@ -52,7 +87,7 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
}
// Validate provider type if set
- if provider.Provider != "" && !isValidProviderType(provider.Provider) {
+ if provider.Provider != "" && !s.validType(provider.Provider) {
return fmt.Errorf("invalid provider type: %s", provider.Provider)
}
@@ -63,7 +98,7 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
}
if provider.Provider != "" {
- if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil {
+ if err := s.validateParams(provider.Provider, provider.Parameters); err != nil {
return err
}
}
diff --git a/internal/container/container.go b/internal/container/container.go
index 655616aa..2ea8cea6 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -74,6 +74,8 @@ import (
"github.com/Tencent/WeKnora/internal/plugin/activate"
pluginbuiltin "github.com/Tencent/WeKnora/internal/plugin/builtin"
plugindriver "github.com/Tencent/WeKnora/internal/plugin/driver"
+ pluginhost "github.com/Tencent/WeKnora/internal/plugin/host"
+ pluginmanifest "github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry"
plugintenancy "github.com/Tencent/WeKnora/internal/plugin/tenancy"
@@ -165,6 +167,10 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(activate.NewMCPServers))
must(container.Provide(activate.NewSkills))
must(container.Provide(activate.NewModelVendors))
+ must(container.Provide(pluginhost.NewManager))
+ must(container.Provide(newPluginInvoker))
+ must(container.Provide(activate.NewWebSearch))
+ must(container.Provide(activate.NewConnectors))
must(container.Provide(newMCPServiceRepository))
must(container.Provide(repository.NewMCPToolApprovalRepository))
must(container.Provide(repository.NewMCPOAuthRepository))
@@ -1835,7 +1841,8 @@ func newPluginRegistry(
// declarative packages today; host, remote and kubernetes drivers join this
// set.
func newPluginDrivers(reg *pluginregistry.Registry, r *reconcile.Reconciler) *plugindriver.Set {
- return plugindriver.NewSet(plugindriver.NewBuiltin(reg.Plugin), r.Driver())
+ return plugindriver.NewSet(plugindriver.NewBuiltin(reg.Plugin),
+ r.Driver(pluginmanifest.RuntimeDeclarative), r.Driver(pluginmanifest.RuntimeHost))
}
// installPluginGate gives the integration handlers the tenant plugin switches,
diff --git a/internal/container/plugin_runtime.go b/internal/container/plugin_runtime.go
index 133b9746..fc84f88e 100644
--- a/internal/container/plugin_runtime.go
+++ b/internal/container/plugin_runtime.go
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/handler"
"github.com/Tencent/WeKnora/internal/plugin/activate"
+ "github.com/Tencent/WeKnora/internal/plugin/host"
"github.com/Tencent/WeKnora/internal/plugin/install"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
pluginregistry "github.com/Tencent/WeKnora/internal/plugin/registry"
@@ -34,13 +35,19 @@ func newPluginPackageStore(cfg *config.Config) (reconcile.PackageStore, error) {
type pluginActivators struct {
dig.In
- MCP *activate.MCPServers
- Skills *activate.Skills
- Vendors *activate.ModelVendors
+ Host *host.Manager
+ WebSearch *activate.WebSearch
+ Connectors *activate.Connectors
+ MCP *activate.MCPServers
+ Skills *activate.Skills
+ Vendors *activate.ModelVendors
+ Invoker *activate.Invoker
}
+// list orders the activators: the host first, so a code plugin's process is
+// running before anything routes calls to it.
func (a pluginActivators) list() []reconcile.Activator {
- return []reconcile.Activator{a.Vendors, a.MCP, a.Skills}
+ return []reconcile.Activator{a.Host, a.WebSearch, a.Connectors, a.Vendors, a.MCP, a.Skills}
}
// bindPluginActivators hands the activators what they need once the plugin
@@ -50,10 +57,14 @@ func bindPluginActivators(
skills *service.TenantSkillService,
) {
a.MCP.Bind(t, repo)
+ a.Invoker.Bind(t, repo)
a.Skills.Bind(t)
skills.SetPluginSkills(a.Skills)
}
+// newPluginInvoker reaches code plugins through this node's plugin host.
+func newPluginInvoker(h *host.Manager) *activate.Invoker { return activate.NewInvoker(h) }
+
func newMCPServiceRepository(db *gorm.DB, plugins *activate.MCPServers) interfaces.MCPServiceRepository {
return plugins.Repository(repository.NewMCPServiceRepository(db))
}
@@ -80,8 +91,13 @@ func newPluginInstaller(
// startPluginReconciler loads installed plugins before the server takes
// traffic, then keeps this node in step with the others.
-func startPluginReconciler(r *reconcile.Reconciler, cleaner interfaces.ResourceCleaner) {
+func startPluginReconciler(r *reconcile.Reconciler, hostManager *host.Manager, cleaner interfaces.ResourceCleaner) {
+ hostManager.SetReporter(r)
ctx, cancel := context.WithCancel(context.Background())
r.Start(ctx)
- cleaner.RegisterWithName("PluginReconciler", func() error { cancel(); return nil })
+ cleaner.RegisterWithName("PluginReconciler", func() error {
+ cancel()
+ hostManager.Close()
+ return nil
+ })
}
diff --git a/internal/datasource/connector.go b/internal/datasource/connector.go
index fa49aeb6..fba14661 100644
--- a/internal/datasource/connector.go
+++ b/internal/datasource/connector.go
@@ -2,8 +2,10 @@ package datasource
import (
"context"
+ "fmt"
"math"
"sort"
+ "sync"
"github.com/Tencent/WeKnora/internal/plugin/configschema"
"github.com/Tencent/WeKnora/internal/types"
@@ -121,15 +123,21 @@ type FullSyncWithCursor interface {
) ([]types.FetchedItem, *types.SyncCursor, error)
}
-// ConnectorRegistry manages the registration and lookup of available connectors
+// ConnectorRegistry manages the registration and lookup of available
+// connectors. Installed plugins add and remove connectors while the server
+// runs, so it is safe for concurrent use.
type ConnectorRegistry struct {
+ mu sync.RWMutex
connectors map[string]Connector
+ // plugins holds the metadata of connectors installed plugins registered.
+ plugins map[string]ConnectorMetadata
}
// NewConnectorRegistry creates a new connector registry
func NewConnectorRegistry() *ConnectorRegistry {
return &ConnectorRegistry{
connectors: make(map[string]Connector),
+ plugins: make(map[string]ConnectorMetadata),
}
}
@@ -141,12 +149,51 @@ func (r *ConnectorRegistry) Register(connector Connector) error {
if connector.Type() == "" {
return ErrConnectorTypeEmpty
}
+ r.mu.Lock()
+ defer r.mu.Unlock()
r.connectors[connector.Type()] = connector
return nil
}
+// RegisterPlugin adds or replaces a plugin's connector with the metadata it
+// is listed under. It refuses to shadow a builtin connector.
+func (r *ConnectorRegistry) RegisterPlugin(connector Connector, meta ConnectorMetadata) error {
+ if connector == nil {
+ return ErrConnectorNil
+ }
+ t := connector.Type()
+ if t == "" {
+ return ErrConnectorTypeEmpty
+ }
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if _, exists := r.connectors[t]; exists {
+ if _, isPlugin := r.plugins[t]; !isPlugin {
+ return fmt.Errorf("connector type %s already exists", t)
+ }
+ }
+ meta.Type = t
+ r.connectors[t] = connector
+ r.plugins[t] = meta
+ return nil
+}
+
+// Unregister removes a plugin's connector; builtins stay. Syncs already
+// running keep the connector they started with.
+func (r *ConnectorRegistry) Unregister(connectorType string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if _, isPlugin := r.plugins[connectorType]; !isPlugin {
+ return
+ }
+ delete(r.plugins, connectorType)
+ delete(r.connectors, connectorType)
+}
+
// Get retrieves a connector by type
func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
connector, exists := r.connectors[connectorType]
if !exists {
return nil, ErrConnectorNotFound
@@ -156,6 +203,8 @@ func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) {
// List returns all registered connector types
func (r *ConnectorRegistry) List() []string {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
types := make([]string, 0, len(r.connectors))
for t := range r.connectors {
types = append(types, t)
@@ -182,6 +231,16 @@ type ConnectorMetadata struct {
// ConfigSchema describes DataSourceConfig.Credentials for this
// connector; the editor renders the credential form from it.
ConfigSchema *configschema.Schema `json:"config_schema,omitempty"`
+ // SettingsSchema describes DataSourceConfig.Settings (non-secret
+ // options) for connectors whose settings form is not built into the
+ // frontend, such as plugin connectors.
+ SettingsSchema *configschema.Schema `json:"settings_schema,omitempty"`
+ // Names and Descriptions localize Name and Description, keyed by
+ // locale, for plugin connectors (builtins use frontend locale keys).
+ Names map[string]string `json:"names,omitempty"`
+ Descriptions map[string]string `json:"descriptions,omitempty"`
+ // PluginID names the installed plugin providing the connector.
+ PluginID string `json:"plugin_id,omitempty"`
}
// GetConnectorMetadata returns metadata for all available connectors
@@ -346,6 +405,8 @@ func ListAvailableConnectors() []ConnectorMetadata {
// ListAvailableConnectors order. A registered connector without an entry in
// ConnectorMetadataRegistry is listed last under its type.
func (r *ConnectorRegistry) Metadata() []ConnectorMetadata {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
out := make([]ConnectorMetadata, 0, len(r.connectors))
seen := make(map[string]bool, len(r.connectors))
for _, meta := range ListAvailableConnectors() {
@@ -362,6 +423,10 @@ func (r *ConnectorRegistry) Metadata() []ConnectorMetadata {
}
sort.Strings(rest)
for _, t := range rest {
+ if meta, ok := r.plugins[t]; ok {
+ out = append(out, meta)
+ continue
+ }
out = append(out, ConnectorMetadata{Type: t, Name: t, Priority: math.MaxInt32})
}
return out
diff --git a/internal/handler/datasource.go b/internal/handler/datasource.go
index 19e3445b..d84aaad1 100644
--- a/internal/handler/datasource.go
+++ b/internal/handler/datasource.go
@@ -326,13 +326,18 @@ func (h *DataSourceHandler) ValidateCredentials(c *gin.Context) {
var req struct {
Type string `json:"type" binding:"required"`
Credentials map[string]interface{} `json:"credentials" binding:"required"`
+ Settings map[string]interface{} `json:"settings"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: type and credentials are required"})
return
}
+ if !h.pluginFilter(c)(manifest.PointConnectors, req.Type) {
+ c.JSON(http.StatusBadRequest, gin.H{"error": disabledIntegrationError})
+ return
+ }
- if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials); err != nil {
+ if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials, req.Settings); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
diff --git a/internal/handler/web_search_provider.go b/internal/handler/web_search_provider.go
index ae7f60a4..a7908351 100644
--- a/internal/handler/web_search_provider.go
+++ b/internal/handler/web_search_provider.go
@@ -329,6 +329,9 @@ func (h *WebSearchProviderHandler) DeleteProvider(c *gin.Context) {
func (h *WebSearchProviderHandler) ListProviderTypes(c *gin.Context) {
enabled := h.pluginFilter(c)
all := types.GetWebSearchProviderTypes()
+ if h.registry != nil {
+ all = append(all, h.registry.PluginTypes()...)
+ }
out := make([]types.WebSearchProviderTypeInfo, 0, len(all))
for _, info := range all {
if enabled(manifest.PointWebSearch, info.ID) {
@@ -408,6 +411,11 @@ func (h *WebSearchProviderHandler) TestProviderRaw(c *gin.Context) {
return
}
+ if !h.pluginFilter(c)(manifest.PointWebSearch, req.Provider) {
+ _ = c.Error(errors.NewBadRequestError(disabledIntegrationError))
+ return
+ }
+
if err := h.doTestSearch(ctx, req.Provider, req.Parameters); err != nil {
logger.Warnf(ctx, "Web search provider test failed: %v", err)
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
diff --git a/internal/infrastructure/web_search/registry.go b/internal/infrastructure/web_search/registry.go
index 0e2ea696..fcadc668 100644
--- a/internal/infrastructure/web_search/registry.go
+++ b/internal/infrastructure/web_search/registry.go
@@ -17,16 +17,74 @@ type ProviderFactory func(params types.WebSearchProviderParameters) (interfaces.
// Instances are created on-demand with tenant-specific parameters.
type Registry struct {
factories map[string]ProviderFactory
- mu sync.RWMutex
+ // plugins describes the types installed plugins registered; builtin
+ // types are described by types.GetWebSearchProviderTypes.
+ plugins map[string]PluginType
+ mu sync.RWMutex
+}
+
+// PluginType is a provider type an installed plugin contributes.
+type PluginType struct {
+ Info types.WebSearchProviderTypeInfo
+ // Validate checks instance parameters against the plugin's schema.
+ Validate func(params types.WebSearchProviderParameters) error
}
// NewRegistry creates a new web search provider registry
func NewRegistry() *Registry {
return &Registry{
factories: make(map[string]ProviderFactory),
+ plugins: make(map[string]PluginType),
}
}
+// RegisterPlugin adds or replaces a plugin's provider type. It refuses to
+// shadow a builtin type.
+func (r *Registry) RegisterPlugin(t PluginType, factory ProviderFactory) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ id := t.Info.ID
+ if _, exists := r.factories[id]; exists {
+ if _, isPlugin := r.plugins[id]; !isPlugin {
+ return fmt.Errorf("web search provider type %s already exists", id)
+ }
+ }
+ r.factories[id] = factory
+ r.plugins[id] = t
+ return nil
+}
+
+// Unregister removes a plugin's provider type; builtins stay.
+func (r *Registry) Unregister(id string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if _, isPlugin := r.plugins[id]; !isPlugin {
+ return
+ }
+ delete(r.plugins, id)
+ delete(r.factories, id)
+}
+
+// PluginType returns a plugin provider type.
+func (r *Registry) PluginType(id string) (PluginType, bool) {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
+ t, ok := r.plugins[id]
+ return t, ok
+}
+
+// PluginTypes returns the plugin provider types, sorted by ID.
+func (r *Registry) PluginTypes() []types.WebSearchProviderTypeInfo {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
+ out := make([]types.WebSearchProviderTypeInfo, 0, len(r.plugins))
+ for _, t := range r.plugins {
+ out = append(out, t.Info)
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+ return out
+}
+
// Register registers a provider type factory by ID
func (r *Registry) Register(id string, factory ProviderFactory) {
r.mu.Lock()
diff --git a/internal/plugin/activate/connectors.go b/internal/plugin/activate/connectors.go
new file mode 100644
index 00000000..f0464026
--- /dev/null
+++ b/internal/plugin/activate/connectors.go
@@ -0,0 +1,354 @@
+package activate
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/plugin/configschema"
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
+ "github.com/Tencent/WeKnora/internal/plugin/reconcile"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// Connectors registers code plugins' data source connectors. A plugin
+// connector's instanceSchema splits in two: fields in x-group "settings"
+// are non-secret options (DataSourceConfig.Settings), everything else is
+// credentials (encrypted at rest).
+type Connectors struct {
+ iv *Invoker
+ registry *datasource.ConnectorRegistry
+
+ mu sync.Mutex
+ registered map[string][]string
+}
+
+// NewConnectors creates the connector activator.
+func NewConnectors(iv *Invoker, registry *datasource.ConnectorRegistry) *Connectors {
+ return &Connectors{iv: iv, registry: registry, registered: map[string][]string{}}
+}
+
+// Name implements reconcile.Activator.
+func (a *Connectors) Name() string { return "connectors" }
+
+// settingsGroup marks instanceSchema fields that are settings, not
+// credentials.
+const settingsGroup = "settings"
+
+// Activate implements reconcile.Activator: all of a plugin's connectors or
+// none.
+func (a *Connectors) Activate(_ context.Context, l *reconcile.Loaded) error {
+ var ids []string
+ for _, c := range l.Manifest.Contributes[manifest.PointConnectors] {
+ id := manifest.QualifiedID(l.Manifest.ID, c.ID)
+ meta, err := connectorMetadata(l, c, id)
+ if err == nil {
+ rc := &remoteConnector{iv: a.iv, m: l.Manifest, local: c.ID, typeID: id}
+ err = a.registry.RegisterPlugin(rc, meta)
+ }
+ if err != nil {
+ for _, done := range ids {
+ a.registry.Unregister(done)
+ }
+ return fmt.Errorf("connector %s: %w", c.ID, err)
+ }
+ ids = append(ids, id)
+ }
+ a.mu.Lock()
+ a.registered[l.Manifest.ID] = ids
+ a.mu.Unlock()
+ return nil
+}
+
+// Deactivate implements reconcile.Activator.
+func (a *Connectors) Deactivate(_ context.Context, pluginID string) error {
+ a.mu.Lock()
+ ids := a.registered[pluginID]
+ delete(a.registered, pluginID)
+ a.mu.Unlock()
+ for _, id := range ids {
+ a.registry.Unregister(id)
+ }
+ return nil
+}
+
+func connectorMetadata(
+ l *reconcile.Loaded,
+ c manifest.Contribution,
+ id string,
+) (datasource.ConnectorMetadata, error) {
+ src, err := instanceSchema(l.Package, c)
+ if err != nil {
+ return datasource.ConnectorMetadata{}, err
+ }
+ creds, settings := splitConnectorSchema(src)
+ icon, err := iconDataURI(l.Package, c.Icon)
+ if err != nil {
+ return datasource.ConnectorMetadata{}, err
+ }
+ caps := c.Capabilities
+ if caps == nil {
+ caps = []string{}
+ }
+ return datasource.ConnectorMetadata{
+ Type: id, Name: c.Name.Default, Names: c.Name.Locales,
+ Description: c.Description.Default, Descriptions: c.Description.Locales,
+ Icon: icon, Priority: 1000 + c.Order, AuthType: "plugin", Capabilities: caps,
+ ConfigSchema: creds, SettingsSchema: settings, PluginID: l.Manifest.ID,
+ }, nil
+}
+
+// splitConnectorSchema separates settings fields from credential fields.
+func splitConnectorSchema(src *configschema.Schema) (creds, settings *configschema.Schema) {
+ if src == nil {
+ return nil, nil
+ }
+ required := map[string]bool{}
+ for _, k := range src.Required {
+ required[k] = true
+ }
+ creds, settings = configschema.Object(), configschema.Object()
+ keys := make([]string, 0, len(src.Properties))
+ for k := range src.Properties {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ prop := *src.Properties[k]
+ if prop.Group == settingsGroup && !prop.Secret {
+ prop.Group = ""
+ settings.Set(k, &prop, required[k])
+ continue
+ }
+ creds.Set(k, &prop, required[k])
+ }
+ if len(creds.Properties) == 0 {
+ creds = nil
+ }
+ if len(settings.Properties) == 0 {
+ settings = nil
+ }
+ return creds, settings
+}
+
+// remoteConnector is a plugin connector. It streams, so the sync service
+// interleaves fetch, ingest and checkpoint and resumes after a timeout.
+type remoteConnector struct {
+ iv *Invoker
+ m *manifest.Manifest
+ local string
+ typeID string
+}
+
+var _ datasource.FullStreamingConnector = (*remoteConnector)(nil)
+
+func (r *remoteConnector) Type() string { return r.typeID }
+
+func instanceOf(cfg *types.DataSourceConfig) map[string]any {
+ if cfg == nil {
+ return map[string]any{}
+ }
+ return map[string]any{
+ "credentials": nonNil(cfg.Credentials),
+ "settings": nonNil(cfg.Settings),
+ "resourceIds": cfg.ResourceIDs,
+ }
+}
+
+func nonNil(m map[string]any) map[string]any {
+ if m == nil {
+ return map[string]any{}
+ }
+ return m
+}
+
+// Validate implements datasource.Connector.
+func (r *remoteConnector) Validate(ctx context.Context, cfg *types.DataSourceConfig) error {
+ ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
+ defer cancel()
+ return r.iv.Call(ctx, r.m, pluginapi.ConnectorValidatePath(r.local), instanceOf(cfg), nil, nil)
+}
+
+// ListResources implements datasource.Connector.
+func (r *remoteConnector) ListResources(
+ ctx context.Context, cfg *types.DataSourceConfig, parentID string,
+) ([]types.Resource, error) {
+ ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
+ defer cancel()
+ var out pluginapi.ListResourcesOutput
+ err := r.iv.Call(ctx, r.m, pluginapi.ConnectorListResourcesPath(r.local), instanceOf(cfg),
+ pluginapi.ListResourcesInput{ParentID: parentID}, &out)
+ if err != nil {
+ return nil, err
+ }
+ res := make([]types.Resource, 0, len(out.Resources))
+ for _, x := range out.Resources {
+ item := types.Resource{
+ ExternalID: x.ExternalID, Name: x.Name, Type: x.Type, Description: x.Description, URL: x.URL,
+ ParentID: x.ParentID, HasChildren: x.HasChildren, Metadata: x.Metadata,
+ }
+ if x.ModifiedAt != nil {
+ item.ModifiedAt = *x.ModifiedAt
+ }
+ res = append(res, item)
+ }
+ return res, nil
+}
+
+// ResolveResourceAncestors implements datasource.Connector.
+func (r *remoteConnector) ResolveResourceAncestors(
+ ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
+) ([]string, error) {
+ ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
+ defer cancel()
+ var out pluginapi.ResolveAncestorsOutput
+ err := r.iv.Call(ctx, r.m, pluginapi.ConnectorResolveAncestorsPath(r.local), instanceOf(cfg),
+ pluginapi.ResolveAncestorsInput{ResourceIDs: resourceIDs}, &out)
+ return out.Ancestors, err
+}
+
+// collector gathers a stream for the non-streaming entry points.
+type collector struct {
+ items []types.FetchedItem
+ cursor *types.SyncCursor
+}
+
+func (c *collector) Emit(_ context.Context, item types.FetchedItem) error {
+ c.items = append(c.items, item)
+ return nil
+}
+
+func (c *collector) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
+ c.cursor = cursor
+ return nil
+}
+
+// FetchAll implements datasource.Connector.
+func (r *remoteConnector) FetchAll(
+ ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
+) ([]types.FetchedItem, error) {
+ c := &collector{}
+ scoped := *cfg
+ if len(resourceIDs) > 0 {
+ scoped.ResourceIDs = resourceIDs
+ }
+ _, err := r.fetch(ctx, &scoped, pluginapi.FetchFull, nil, c)
+ return c.items, err
+}
+
+// FetchIncremental implements datasource.Connector.
+func (r *remoteConnector) FetchIncremental(
+ ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor,
+) ([]types.FetchedItem, *types.SyncCursor, error) {
+ c := &collector{}
+ next, err := r.fetch(ctx, cfg, pluginapi.FetchIncremental, cursor, c)
+ return c.items, next, err
+}
+
+// FetchStream implements datasource.StreamingConnector: incremental from a
+// cursor, full without one.
+func (r *remoteConnector) FetchStream(
+ ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
+) (*types.SyncCursor, error) {
+ mode := pluginapi.FetchIncremental
+ if cursor == nil {
+ mode = pluginapi.FetchFull
+ }
+ return r.fetch(ctx, cfg, mode, cursor, h)
+}
+
+// FetchFullStream implements datasource.FullStreamingConnector: every item
+// again, with the previous cursor for deletion reconciliation.
+func (r *remoteConnector) FetchFullStream(
+ ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
+) (*types.SyncCursor, error) {
+ return r.fetch(ctx, cfg, pluginapi.FetchFull, cursor, h)
+}
+
+func (r *remoteConnector) fetch(
+ ctx context.Context, cfg *types.DataSourceConfig, mode string, cursor *types.SyncCursor,
+ h datasource.StreamHandler,
+) (*types.SyncCursor, error) {
+ in := pluginapi.FetchInput{Mode: mode, Cursor: toPluginCursor(cursor)}
+ if cfg != nil {
+ in.ResourceIDs = cfg.ResourceIDs
+ }
+ end, err := r.iv.Stream(ctx, r.m, pluginapi.ConnectorFetchPath(r.local), instanceOf(cfg), in,
+ func(ev pluginapi.Event) error {
+ switch ev.Type {
+ case pluginapi.EventItem:
+ var it pluginapi.FetchedItem
+ if err := json.Unmarshal(ev.Data, &it); err != nil {
+ return fmt.Errorf("decode fetched item: %w", err)
+ }
+ return h.Emit(ctx, fromPluginItem(it))
+ case pluginapi.EventCheckpoint:
+ var c pluginapi.Cursor
+ if err := json.Unmarshal(ev.Data, &c); err != nil {
+ return fmt.Errorf("decode checkpoint: %w", err)
+ }
+ return h.Checkpoint(ctx, fromPluginCursor(&c))
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ var final pluginapi.Cursor
+ if len(end) > 0 {
+ if err := json.Unmarshal(end, &final); err != nil {
+ return nil, fmt.Errorf("decode final cursor: %w", err)
+ }
+ }
+ next := fromPluginCursor(&final)
+ if next.LastSyncTime.IsZero() {
+ next.LastSyncTime = time.Now()
+ }
+ return next, nil
+}
+
+func toPluginCursor(c *types.SyncCursor) *pluginapi.Cursor {
+ if c == nil {
+ return nil
+ }
+ out := &pluginapi.Cursor{State: c.ConnectorCursor}
+ if !c.LastSyncTime.IsZero() {
+ t := c.LastSyncTime
+ out.LastSyncTime = &t
+ }
+ return out
+}
+
+func fromPluginCursor(c *pluginapi.Cursor) *types.SyncCursor {
+ out := &types.SyncCursor{ConnectorCursor: c.State}
+ if c.LastSyncTime != nil {
+ out.LastSyncTime = *c.LastSyncTime
+ }
+ if out.ConnectorCursor == nil {
+ out.ConnectorCursor = map[string]any{}
+ }
+ return out
+}
+
+func fromPluginItem(it pluginapi.FetchedItem) types.FetchedItem {
+ out := types.FetchedItem{
+ ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType,
+ FileName: it.FileName, URL: it.URL, Metadata: it.Metadata, IsDeleted: it.IsDeleted,
+ SourceResourceID: it.SourceResourceID,
+ }
+ if it.UpdatedAt != nil {
+ out.UpdatedAt = *it.UpdatedAt
+ }
+ if it.CreatedAt != nil {
+ out.CreatedAt = *it.CreatedAt
+ }
+ if out.ContentType == "" {
+ out.ContentType = "text/markdown"
+ }
+ return out
+}
diff --git a/internal/plugin/activate/connectors_test.go b/internal/plugin/activate/connectors_test.go
new file mode 100644
index 00000000..3fbe184b
--- /dev/null
+++ b/internal/plugin/activate/connectors_test.go
@@ -0,0 +1,202 @@
+package activate
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/plugin/pkg"
+ "github.com/Tencent/WeKnora/internal/plugin/plugintest"
+ "github.com/Tencent/WeKnora/internal/plugin/reconcile"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/pluginsdk"
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+const feedPluginManifest = `schemaVersion: 1
+id: acme.feeds
+version: 1.0.0
+apiVersion: weknora.plugin/v1
+name: { en-US: ACME Feeds }
+publisher: { id: acme }
+runtime: { type: host, kind: binary, entry: bin/feeds }
+contributes:
+ connectors:
+ - id: feed
+ name: { en-US: ACME Feed, zh-CN: ACME 订阅 }
+ capabilities: [incremental]
+ instanceSchema: schemas/feed.yaml
+`
+
+const feedSchema = `type: object
+required: [feed_urls]
+properties:
+ token: { type: string, x-secret: true }
+ feed_urls: { type: string, x-widget: textarea, x-group: settings }
+`
+
+// feedPlugin serves numbered items and resumes from the cursor.
+type feedPlugin struct{}
+
+func (feedPlugin) Validate(_ context.Context, _ *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
+ if cfg.Settings["feed_urls"] == nil {
+ return pluginapi.InvalidConfig("feed_urls is required", map[string]string{"settings.feed_urls": "required"})
+ }
+ if cfg.Credentials["token"] != "t-1" {
+ return pluginapi.Errorf(pluginapi.CodeUnauthorized, "bad token")
+ }
+ return nil
+}
+
+func (feedPlugin) ListResources(
+ _ context.Context,
+ _ *pluginsdk.Call,
+ cfg pluginsdk.ConnectorConfig,
+ parent string,
+) ([]pluginapi.Resource, error) {
+ if parent != "" {
+ return nil, nil
+ }
+ var out []pluginapi.Resource
+ for _, u := range strings.Split(fmt.Sprint(cfg.Settings["feed_urls"]), "\n") {
+ out = append(out, pluginapi.Resource{ExternalID: u, Name: u, Type: "feed"})
+ }
+ return out, nil
+}
+
+func (feedPlugin) Fetch(
+ _ context.Context,
+ _ *pluginsdk.Call,
+ _ pluginsdk.ConnectorConfig,
+ in pluginapi.FetchInput,
+ s *pluginsdk.Stream,
+) (*pluginapi.Cursor, error) {
+ start := 0
+ if in.Mode == pluginapi.FetchIncremental && in.Cursor != nil {
+ start = int(in.Cursor.State["seen"].(float64))
+ }
+ for i := start; i < 3; i++ {
+ item := pluginapi.FetchedItem{
+ ExternalID: fmt.Sprint("post-", i), Title: fmt.Sprint("Post ", i), Content: []byte("# hi"),
+ }
+ if err := s.Item(item); err != nil {
+ return nil, err
+ }
+ if err := s.Checkpoint(pluginapi.Cursor{State: map[string]any{"seen": i + 1}}); err != nil {
+ return nil, err
+ }
+ }
+ return &pluginapi.Cursor{State: map[string]any{"seen": 3}}, nil
+}
+
+type recordingHandler struct {
+ items []string
+ checkpoints int
+}
+
+func (h *recordingHandler) Emit(_ context.Context, item types.FetchedItem) error {
+ h.items = append(h.items, item.ExternalID+"|"+item.ContentType)
+ return nil
+}
+
+func (h *recordingHandler) Checkpoint(context.Context, *types.SyncCursor) error {
+ h.checkpoints++
+ return nil
+}
+
+func TestPluginConnector(t *testing.T) {
+ ctx := context.Background()
+ plugin := pluginsdk.New(pluginsdk.Info{ID: "acme.feeds", Version: "1.0.0"})
+ plugin.Connector("feed", feedPlugin{})
+ srv := httptest.NewServer(plugin.Handler())
+ defer srv.Close()
+
+ p, err := pkg.Open(plugintest.Zip(t, map[string]string{
+ "plugin.yaml": feedPluginManifest, "bin/feeds": "x", "schemas/feed.yaml": feedSchema,
+ }))
+ if err != nil {
+ t.Fatal(err)
+ }
+ registry := datasource.NewConnectorRegistry()
+ a := NewConnectors(NewInvoker(fakeClients{client.New(srv.URL, nil, nil)}), registry)
+ if err := a.Activate(ctx, &reconcile.Loaded{Manifest: p.Manifest, Package: p}); err != nil {
+ t.Fatalf("Activate: %v", err)
+ }
+
+ var meta datasource.ConnectorMetadata
+ for _, m := range registry.Metadata() {
+ if m.Type == "acme.feeds/feed" {
+ meta = m
+ }
+ }
+ if meta.Names["zh-CN"] != "ACME 订阅" || meta.PluginID != "acme.feeds" ||
+ meta.ConfigSchema.Properties["token"] == nil || meta.SettingsSchema.Properties["feed_urls"] == nil ||
+ meta.ConfigSchema.Properties["feed_urls"] != nil {
+ t.Fatalf("metadata = %+v", meta)
+ }
+
+ conn, err := registry.Get("acme.feeds/feed")
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg := &types.DataSourceConfig{
+ Credentials: map[string]any{"token": "t-1"},
+ Settings: map[string]any{"feed_urls": "https://a.example/rss\nhttps://b.example/rss"},
+ }
+ err = conn.Validate(ctx, &types.DataSourceConfig{Credentials: map[string]any{"token": "t-1"}})
+ var pe *pluginapi.Error
+ if err == nil || err.Error() != "feed_urls is required" ||
+ !errors.As(err, &pe) || pe.Code != pluginapi.CodeInvalidConfig {
+ t.Fatalf("Validate without settings = %v; want the plugin's message with its code reachable", err)
+ }
+ if err := conn.Validate(ctx, cfg); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+ res, err := conn.ListResources(ctx, cfg, "")
+ if err != nil || len(res) != 2 || res[1].ExternalID != "https://b.example/rss" {
+ t.Fatalf("ListResources = %+v, %v", res, err)
+ }
+
+ streaming := conn.(datasource.StreamingConnector)
+ h := &recordingHandler{}
+ cursor, err := streaming.FetchStream(ctx, cfg, nil, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Join(h.items, ",") != "post-0|text/markdown,post-1|text/markdown,post-2|text/markdown" ||
+ h.checkpoints != 3 || cursor.ConnectorCursor["seen"].(float64) != 3 || cursor.LastSyncTime.IsZero() {
+ t.Fatalf("items=%v checkpoints=%d cursor=%+v", h.items, h.checkpoints, cursor)
+ }
+ cursor.ConnectorCursor["seen"] = float64(2)
+ items, next, err := conn.FetchIncremental(ctx, cfg, cursor)
+ if err != nil || len(items) != 1 || items[0].ExternalID != "post-2" ||
+ next.ConnectorCursor["seen"].(float64) != 3 {
+ t.Fatalf("incremental = %+v %+v %v", items, next, err)
+ }
+
+ if err := a.Deactivate(ctx, "acme.feeds"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := registry.Get("acme.feeds/feed"); err == nil {
+ t.Fatal("connector survived Deactivate")
+ }
+}
+
+func TestPluginConnectorCannotShadowABuiltin(t *testing.T) {
+ registry := datasource.NewConnectorRegistry()
+ if err := registry.Register(&remoteConnector{typeID: "rss"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := registry.RegisterPlugin(&remoteConnector{typeID: "rss"}, datasource.ConnectorMetadata{}); err == nil {
+ t.Fatal("a plugin must not replace a builtin connector")
+ }
+ registry.Unregister("rss")
+ if _, err := registry.Get("rss"); err != nil {
+ t.Fatal("Unregister must not remove builtins")
+ }
+}
diff --git a/internal/plugin/activate/invoke.go b/internal/plugin/activate/invoke.go
new file mode 100644
index 00000000..82617aa9
--- /dev/null
+++ b/internal/plugin/activate/invoke.go
@@ -0,0 +1,153 @@
+package activate
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/plugin/install"
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
+ "github.com/Tencent/WeKnora/internal/plugin/tenancy"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// ClientSource reaches running code plugins (the node's plugin host).
+type ClientSource interface {
+ Client(pluginID string) (*client.Client, error)
+}
+
+// Invoker is how the remote adapters call a code plugin: it finds the
+// process and builds the protocol envelope from the request context and the
+// plugin's decrypted system and tenant configuration.
+type Invoker struct {
+ clients ClientSource
+
+ mu sync.RWMutex
+ tenancy *tenancy.Service
+ plugins interfaces.PluginRepository
+}
+
+// NewInvoker creates an Invoker; Bind completes it.
+func NewInvoker(clients ClientSource) *Invoker { return &Invoker{clients: clients} }
+
+// Bind supplies the tenant configuration and installed plugin rows.
+func (iv *Invoker) Bind(t *tenancy.Service, plugins interfaces.PluginRepository) {
+ iv.mu.Lock()
+ iv.tenancy, iv.plugins = t, plugins
+ iv.mu.Unlock()
+}
+
+// Default timeouts per call kind, used when the caller set no deadline.
+const (
+ searchTimeout = 10 * time.Second
+ metadataTimeout = time.Minute
+)
+
+// withDefaultTimeout bounds ctx unless it already has a deadline.
+func withDefaultTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) {
+ if _, ok := ctx.Deadline(); ok {
+ return context.WithCancel(ctx)
+ }
+ return context.WithTimeout(ctx, d)
+}
+
+// Envelope builds the call envelope for a plugin with an instance config.
+func (iv *Invoker) Envelope(
+ ctx context.Context,
+ m *manifest.Manifest,
+ instance map[string]any,
+) (pluginapi.Envelope, error) {
+ env := pluginapi.Envelope{Config: pluginapi.Config{Instance: instance}}
+ if tenantID, ok := types.TenantIDFromContext(ctx); ok {
+ env.Context.TenantID = tenantID
+ }
+ if userID, ok := ctx.Value(types.UserIDContextKey).(string); ok {
+ env.Context.UserID = userID
+ }
+ if lang, ok := types.LanguageFromContext(ctx); ok {
+ env.Context.Locale = lang
+ }
+ if rid, ok := ctx.Value(types.RequestIDContextKey).(string); ok {
+ env.Context.RequestID = rid
+ }
+ if d, ok := ctx.Deadline(); ok {
+ env.Context.Deadline = &d
+ }
+ iv.mu.RLock()
+ t, plugins := iv.tenancy, iv.plugins
+ iv.mu.RUnlock()
+ if plugins != nil && len(m.Config.SystemSchema) > 0 {
+ sys, _, err := install.OpenSystemConfig(ctx, plugins, m)
+ if err != nil {
+ return env, err
+ }
+ env.Config.System = sys
+ }
+ if t != nil && env.Context.TenantID != 0 && len(m.Config.TenantSchema) > 0 {
+ cfg, _, err := t.OpenConfig(ctx, env.Context.TenantID, m.ID)
+ if err != nil {
+ return env, err
+ }
+ env.Config.Tenant = cfg
+ }
+ return env, nil
+}
+
+// callError presents a plugin's error by its message, the way WeKnora's own
+// errors read in the UI; the protocol code stays reachable through
+// errors.As(err, **pluginapi.Error).
+type callError struct {
+ pluginID string
+ err *pluginapi.Error
+}
+
+func (e *callError) Error() string {
+ if e.err.Code == pluginapi.CodeUnavailable {
+ return "plugin " + e.pluginID + " is unavailable: " + e.err.Message
+ }
+ return e.err.Message
+}
+
+func (e *callError) Unwrap() error { return e.err }
+
+func present(pluginID string, err error) error {
+ if pe, ok := pluginapi.AsError(err); ok {
+ return &callError{pluginID: pluginID, err: pe}
+ }
+ return err
+}
+
+// Call makes a unary call to a plugin.
+func (iv *Invoker) Call(
+ ctx context.Context, m *manifest.Manifest, path string, instance map[string]any, input, out any,
+) error {
+ c, err := iv.clients.Client(m.ID)
+ if err != nil {
+ return present(m.ID, err)
+ }
+ env, err := iv.Envelope(ctx, m, instance)
+ if err != nil {
+ return err
+ }
+ return present(m.ID, c.Call(ctx, path, env, input, out))
+}
+
+// Stream makes a streaming call to a plugin.
+func (iv *Invoker) Stream(
+ ctx context.Context, m *manifest.Manifest, path string, instance map[string]any, input any,
+ fn func(pluginapi.Event) error,
+) ([]byte, error) {
+ c, err := iv.clients.Client(m.ID)
+ if err != nil {
+ return nil, present(m.ID, err)
+ }
+ env, err := iv.Envelope(ctx, m, instance)
+ if err != nil {
+ return nil, err
+ }
+ end, err := c.Stream(ctx, path, env, input, fn)
+ return end, present(m.ID, err)
+}
diff --git a/internal/plugin/activate/websearch.go b/internal/plugin/activate/websearch.go
new file mode 100644
index 00000000..f82d1184
--- /dev/null
+++ b/internal/plugin/activate/websearch.go
@@ -0,0 +1,278 @@
+package activate
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "path"
+ "sort"
+ "strings"
+ "sync"
+
+ "gopkg.in/yaml.v3"
+
+ infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
+ "github.com/Tencent/WeKnora/internal/plugin/configschema"
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
+ "github.com/Tencent/WeKnora/internal/plugin/pkg"
+ "github.com/Tencent/WeKnora/internal/plugin/reconcile"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// WebSearch registers code plugins' web search providers as provider types.
+// A plugin provider stores its configuration in the same parameters as a
+// builtin one: the secret api_key and string fields under extra_config, so
+// encryption, redaction and the credentials endpoint work unchanged.
+type WebSearch struct {
+ iv *Invoker
+ registry *infra_web_search.Registry
+
+ mu sync.Mutex
+ registered map[string][]string // plugin ID → provider type IDs
+}
+
+// NewWebSearch creates the web search activator.
+func NewWebSearch(iv *Invoker, registry *infra_web_search.Registry) *WebSearch {
+ return &WebSearch{iv: iv, registry: registry, registered: map[string][]string{}}
+}
+
+// Name implements reconcile.Activator.
+func (a *WebSearch) Name() string { return "webSearch" }
+
+// Activate implements reconcile.Activator: all of a plugin's providers or
+// none.
+func (a *WebSearch) Activate(_ context.Context, l *reconcile.Loaded) error {
+ var ids []string
+ for _, c := range l.Manifest.Contributes[manifest.PointWebSearch] {
+ id := manifest.QualifiedID(l.Manifest.ID, c.ID)
+ pt, err := webSearchType(l, c, id)
+ if err == nil {
+ m, local := l.Manifest, c.ID
+ err = a.registry.RegisterPlugin(
+ pt,
+ func(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
+ return &remoteSearch{
+ iv: a.iv,
+ m: m,
+ local: local,
+ typeID: id,
+ instance: webSearchInstance(params),
+ }, nil
+ },
+ )
+ }
+ if err != nil {
+ for _, done := range ids {
+ a.registry.Unregister(done)
+ }
+ return fmt.Errorf("webSearch %s: %w", c.ID, err)
+ }
+ ids = append(ids, id)
+ }
+ a.mu.Lock()
+ a.registered[l.Manifest.ID] = ids
+ a.mu.Unlock()
+ return nil
+}
+
+// Deactivate implements reconcile.Activator.
+func (a *WebSearch) Deactivate(_ context.Context, pluginID string) error {
+ a.mu.Lock()
+ ids := a.registered[pluginID]
+ delete(a.registered, pluginID)
+ a.mu.Unlock()
+ for _, id := range ids {
+ a.registry.Unregister(id)
+ }
+ return nil
+}
+
+// webSearchType describes a plugin provider for the type list.
+func webSearchType(l *reconcile.Loaded, c manifest.Contribution, id string) (infra_web_search.PluginType, error) {
+ src, err := instanceSchema(l.Package, c)
+ if err != nil {
+ return infra_web_search.PluginType{}, err
+ }
+ schema, requiresKey, err := webSearchSchema(src)
+ if err != nil {
+ return infra_web_search.PluginType{}, err
+ }
+ icon, err := iconDataURI(l.Package, c.Icon)
+ if err != nil {
+ return infra_web_search.PluginType{}, err
+ }
+ info := types.WebSearchProviderTypeInfo{
+ ID: id, Name: c.Name.Default, Names: c.Name.Locales, Description: c.Description.Default,
+ RequiresAPIKey: requiresKey, Icon: icon, PluginID: l.Manifest.ID, ConfigSchema: schema,
+ }
+ validate := func(params types.WebSearchProviderParameters) error {
+ value := map[string]any{"api_key": params.APIKey}
+ extra := map[string]any{}
+ for k, v := range params.ExtraConfig {
+ extra[k] = v
+ }
+ value["extra_config"] = extra
+ if errs := configschema.Validate(schema, value); len(errs) > 0 {
+ return errs
+ }
+ return nil
+ }
+ return infra_web_search.PluginType{Info: info, Validate: validate}, nil
+}
+
+// webSearchSchema maps a plugin's instance schema onto the provider
+// parameters: api_key stays top-level (the only secret), everything else
+// moves under extra_config, which stores strings.
+func webSearchSchema(src *configschema.Schema) (*configschema.Schema, bool, error) {
+ out := configschema.Object()
+ if src == nil {
+ return out, false, nil
+ }
+ required := map[string]bool{}
+ for _, k := range src.Required {
+ required[k] = true
+ }
+ extra := &configschema.Schema{
+ Type: configschema.TypeObject, Group: "credentials", Order: 40, Properties: map[string]*configschema.Schema{},
+ }
+ var keys []string
+ for k := range src.Properties {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ requiresKey := false
+ for _, k := range keys {
+ prop := *src.Properties[k]
+ if k == "api_key" {
+ prop.Type, prop.Secret, prop.Group = configschema.TypeString, true, "credentials"
+ out.Set("api_key", &prop, required[k])
+ requiresKey = required[k]
+ continue
+ }
+ if prop.Secret {
+ return nil, false, fmt.Errorf("instanceSchema field %s is secret; only api_key may be", k)
+ }
+ if prop.Type != "" && prop.Type != configschema.TypeString {
+ return nil, false, fmt.Errorf(
+ "instanceSchema field %s must be a string (web search settings are stored as strings)",
+ k,
+ )
+ }
+ prop.Type, prop.Group = configschema.TypeString, ""
+ extra.Set(k, &prop, required[k])
+ }
+ if len(extra.Properties) > 0 {
+ out.Set("extra_config", extra, len(extra.Required) > 0)
+ }
+ return out, requiresKey, nil
+}
+
+// webSearchInstance is what the plugin receives as config.instance.
+func webSearchInstance(params types.WebSearchProviderParameters) map[string]any {
+ out := map[string]any{}
+ if params.APIKey != "" {
+ out["api_key"] = params.APIKey
+ }
+ for k, v := range params.ExtraConfig {
+ out[k] = v
+ }
+ return out
+}
+
+// instanceSchema reads a contribution's instanceSchema file (JSON or YAML).
+func instanceSchema(p *pkg.Package, c manifest.Contribution) (*configschema.Schema, error) {
+ if c.InstanceSchema == "" || p == nil {
+ return nil, nil
+ }
+ raw, ok := p.ReadFile(c.InstanceSchema)
+ if !ok {
+ return nil, fmt.Errorf("instanceSchema %s is not in the package", c.InstanceSchema)
+ }
+ var v any
+ if err := yaml.Unmarshal(raw, &v); err != nil {
+ return nil, fmt.Errorf("instanceSchema %s: %w", c.InstanceSchema, err)
+ }
+ b, err := json.Marshal(v)
+ if err != nil {
+ return nil, err
+ }
+ s, err := configschema.Parse(b)
+ if err != nil {
+ return nil, fmt.Errorf("instanceSchema %s: %w", c.InstanceSchema, err)
+ }
+ return s, nil
+}
+
+// maxIconBytes caps an icon inlined into type listings.
+const maxIconBytes = 64 << 10
+
+// iconDataURI inlines a package icon (SVG or PNG) as a data: URI.
+func iconDataURI(p *pkg.Package, file string) (string, error) {
+ if file == "" || p == nil {
+ return "", nil
+ }
+ b, ok := p.ReadFile(file)
+ if !ok {
+ return "", fmt.Errorf("icon %s is not in the package", file)
+ }
+ if len(b) > maxIconBytes {
+ return "", fmt.Errorf("icon %s is over %d bytes", file, maxIconBytes)
+ }
+ var mime string
+ switch strings.ToLower(path.Ext(file)) {
+ case ".svg":
+ mime = "image/svg+xml"
+ case ".png":
+ mime = "image/png"
+ default:
+ return "", fmt.Errorf("icon %s must be .svg or .png", file)
+ }
+ return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(b), nil
+}
+
+// remoteSearch is one configured plugin provider.
+type remoteSearch struct {
+ iv *Invoker
+ m *manifest.Manifest
+ local string
+ typeID string
+ instance map[string]any
+}
+
+// Name implements interfaces.WebSearchProvider.
+func (r *remoteSearch) Name() string { return r.typeID }
+
+// Search implements interfaces.WebSearchProvider.
+func (r *remoteSearch) Search(
+ ctx context.Context, query string, maxResults int, includeDate bool,
+) ([]*types.WebSearchResult, error) {
+ return r.SearchWithFilters(ctx, query, maxResults, includeDate, types.WebSearchFilters{})
+}
+
+// SearchWithFilters implements interfaces.FilteredWebSearchProvider; the
+// plugin must fail rather than ignore a filter it cannot honour.
+func (r *remoteSearch) SearchWithFilters(
+ ctx context.Context, query string, maxResults int, includeDate bool, filters types.WebSearchFilters,
+) ([]*types.WebSearchResult, error) {
+ ctx, cancel := withDefaultTimeout(ctx, searchTimeout)
+ defer cancel()
+ in := pluginapi.SearchInput{
+ Query: query, MaxResults: maxResults, IncludeDate: includeDate,
+ Region: filters.Country, Freshness: filters.Freshness,
+ }
+ var out pluginapi.SearchOutput
+ if err := r.iv.Call(ctx, r.m, pluginapi.SearchPath(r.local), r.instance, in, &out); err != nil {
+ return nil, err
+ }
+ results := make([]*types.WebSearchResult, 0, len(out.Results))
+ for _, h := range out.Results {
+ results = append(results, &types.WebSearchResult{
+ Title: h.Title, URL: h.URL, Snippet: h.Snippet, Content: h.Content,
+ Source: r.typeID, Age: h.Age, PublishedAt: h.PublishedAt,
+ })
+ }
+ return results, nil
+}
diff --git a/internal/plugin/activate/websearch_test.go b/internal/plugin/activate/websearch_test.go
new file mode 100644
index 00000000..d36ac857
--- /dev/null
+++ b/internal/plugin/activate/websearch_test.go
@@ -0,0 +1,152 @@
+package activate
+
+import (
+ "context"
+ "fmt"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
+ "github.com/Tencent/WeKnora/internal/plugin/pkg"
+ "github.com/Tencent/WeKnora/internal/plugin/plugintest"
+ "github.com/Tencent/WeKnora/internal/plugin/reconcile"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/Tencent/WeKnora/pluginsdk"
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+const searchPluginManifest = `schemaVersion: 1
+id: acme.search
+version: 1.0.0
+apiVersion: weknora.plugin/v1
+name: { en-US: ACME Search }
+publisher: { id: acme }
+runtime: { type: host, kind: binary, entry: bin/search }
+contributes:
+ webSearch:
+ - id: brave
+ name: { en-US: ACME Brave, zh-CN: ACME 勇敢搜索 }
+ icon: icons/brave.svg
+ instanceSchema: schemas/brave.yaml
+`
+
+// fakeClients serves one plugin from an in-process SDK handler.
+type fakeClients struct{ c *client.Client }
+
+func (f fakeClients) Client(string) (*client.Client, error) { return f.c, nil }
+
+func searchLoaded(t *testing.T, schema string) *reconcile.Loaded {
+ t.Helper()
+ p, err := pkg.Open(plugintest.Zip(t, map[string]string{
+ "plugin.yaml": searchPluginManifest,
+ "bin/search": "x",
+ "icons/brave.svg": `
`,
+ "schemas/brave.yaml": schema,
+ }))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &reconcile.Loaded{Manifest: p.Manifest, Package: p}
+}
+
+const braveSchema = `type: object
+required: [api_key, market]
+properties:
+ api_key: { type: string, title: API key }
+ market: { type: string, enum: [en-US, zh-CN], default: en-US }
+`
+
+func TestPluginWebSearchProvider(t *testing.T) {
+ ctx := context.Background()
+ plugin := pluginsdk.New(pluginsdk.Info{ID: "acme.search", Version: "1.0.0"})
+ plugin.WebSearch(
+ "brave",
+ pluginsdk.WebSearchFunc(
+ func(_ context.Context, call *pluginsdk.Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
+ if call.Config.Instance["api_key"] != "k-1" || call.Config.Instance["market"] != "zh-CN" {
+ return nil, pluginapi.Errorf(
+ pluginapi.CodeUnauthorized,
+ "bad instance config %v",
+ call.Config.Instance,
+ )
+ }
+ if in.Region != "" && in.Region != "CN" {
+ return nil, pluginapi.Errorf(pluginapi.CodeInvalidConfig, "region %s unsupported", in.Region)
+ }
+ return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{
+ Title: in.Query, URL: fmt.Sprint("https://acme.example/?tenant=", call.TenantID),
+ }}}, nil
+ },
+ ),
+ )
+ srv := httptest.NewServer(plugin.Handler())
+ defer srv.Close()
+
+ registry := infra_web_search.NewRegistry()
+ a := NewWebSearch(NewInvoker(fakeClients{client.New(srv.URL, nil, nil)}), registry)
+ if err := a.Activate(ctx, searchLoaded(t, braveSchema)); err != nil {
+ t.Fatalf("Activate: %v", err)
+ }
+
+ pt, ok := registry.PluginType("acme.search/brave")
+ if !ok {
+ t.Fatal("type not registered")
+ }
+ info := pt.Info
+ if info.Names["zh-CN"] != "ACME 勇敢搜索" || !info.RequiresAPIKey ||
+ !strings.HasPrefix(info.Icon, "data:image/svg+xml;base64,") ||
+ info.PluginID != "acme.search" {
+ t.Fatalf("info = %+v", info)
+ }
+ if ak := info.ConfigSchema.Properties["api_key"]; ak == nil || !ak.Secret || ak.Group != "credentials" {
+ t.Fatalf("api_key schema = %+v", ak)
+ }
+ if ec := info.ConfigSchema.Properties["extra_config"]; ec == nil || ec.Properties["market"] == nil {
+ t.Fatalf("extra_config schema = %+v", ec)
+ }
+ if err := pt.Validate(types.WebSearchProviderParameters{APIKey: "k"}); err == nil {
+ t.Fatal("a missing required extra_config field must fail validation")
+ }
+ params := types.WebSearchProviderParameters{APIKey: "k-1", ExtraConfig: map[string]string{"market": "zh-CN"}}
+ if err := pt.Validate(params); err != nil {
+ t.Fatalf("valid params rejected: %v", err)
+ }
+
+ provider, err := registry.CreateProvider("acme.search/brave", params)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tctx := context.WithValue(ctx, types.TenantIDContextKey, uint64(7))
+ res, err := provider.Search(tctx, "weknora", 3, false)
+ if err != nil || len(res) != 1 || res[0].Title != "weknora" || res[0].URL != "https://acme.example/?tenant=7" ||
+ res[0].Source != "acme.search/brave" {
+ t.Fatalf("search = %+v, %v", res, err)
+ }
+ filtered := provider.(interfaces.FilteredWebSearchProvider)
+ if _, err := filtered.SearchWithFilters(tctx, "q", 3, false, types.WebSearchFilters{Country: "DE"}); err == nil ||
+ !strings.Contains(err.Error(), "region DE unsupported") {
+ t.Fatalf("an unsupported filter must fail, got %v", err)
+ }
+
+ if err := a.Deactivate(ctx, "acme.search"); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := registry.PluginType("acme.search/brave"); ok {
+ t.Fatal("type survived Deactivate")
+ }
+}
+
+func TestPluginWebSearchSchemaRules(t *testing.T) {
+ a := NewWebSearch(NewInvoker(fakeClients{}), infra_web_search.NewRegistry())
+ for name, schema := range map[string]string{
+ "secret field": "type: object\nproperties: { token: { type: string, x-secret: true } }\n",
+ "number field": "type: object\nproperties: { limit: { type: integer } }\n",
+ } {
+ if err := a.Activate(context.Background(), searchLoaded(t, schema)); err == nil {
+ t.Errorf("%s: want an activation error", name)
+ }
+ }
+}
diff --git a/internal/plugin/host/host.go b/internal/plugin/host/host.go
new file mode 100644
index 00000000..450dd77d
--- /dev/null
+++ b/internal/plugin/host/host.go
@@ -0,0 +1,140 @@
+// Package host is the embedded plugin host: it runs host-runtime plugins as
+// child processes of this WeKnora node and hands out clients to reach them.
+// Each node runs its own copy of every enabled host plugin (the reconciler
+// activates it everywhere), so a call never leaves the node.
+//
+// A plugin process gets a private socket, a random token, an environment
+// built from scratch (no WeKnora secrets) and an egress proxy limited to the
+// hosts its manifest was granted. The host checks that the process is the
+// installed package, health-checks it and restarts it with backoff.
+package host
+
+import (
+ "context"
+ "fmt"
+ "sync"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
+ "github.com/Tencent/WeKnora/internal/plugin/reconcile"
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// StateReporter learns when a running plugin's health changes, so the
+// cluster view shows a crashed plugin as degraded.
+type StateReporter interface {
+ ReportRuntime(pluginID string, healthy bool, err error)
+}
+
+// Manager runs host plugins. It is a reconcile.Activator and must come
+// before the activators that route calls to the plugins.
+type Manager struct {
+ reporter StateReporter
+
+ mu sync.Mutex
+ procs map[string]*process
+}
+
+// NewManager creates an empty host.
+func NewManager() *Manager { return &Manager{procs: map[string]*process{}} }
+
+// SetReporter wires health changes to the reconciler.
+func (m *Manager) SetReporter(r StateReporter) {
+ m.mu.Lock()
+ m.reporter = r
+ m.mu.Unlock()
+}
+
+// Name implements reconcile.Activator.
+func (m *Manager) Name() string { return "host" }
+
+// ActivatesInPlace implements reconcile.InPlaceActivator: an upgrade starts
+// the new process before the old one stops.
+func (m *Manager) ActivatesInPlace() {}
+
+// Activate starts a host plugin and returns once it is ready. Other runtimes
+// are ignored. A new version replaces the old process only after it started.
+func (m *Manager) Activate(ctx context.Context, l *reconcile.Loaded) error {
+ if l.Manifest.Runtime.Type != manifest.RuntimeHost {
+ return nil
+ }
+ if l.Manifest.Runtime.Kind != "binary" {
+ return fmt.Errorf("runtime.kind %q is not supported by this host yet; only binary", l.Manifest.Runtime.Kind)
+ }
+ id := l.Manifest.ID
+ p, err := startProcess(spec{m: l.Manifest, dir: l.Dir}, func(s State, err error) {
+ m.report(id, s, err)
+ })
+ if err != nil {
+ return err
+ }
+ m.mu.Lock()
+ old := m.procs[id]
+ m.procs[id] = p
+ m.mu.Unlock()
+ if old != nil {
+ old.stop()
+ }
+ logger.Infof(ctx, "[plugin] host started %s %s", id, l.Manifest.Version)
+ return nil
+}
+
+// Deactivate implements reconcile.Activator: it stops the process.
+func (m *Manager) Deactivate(ctx context.Context, pluginID string) error {
+ m.mu.Lock()
+ p := m.procs[pluginID]
+ delete(m.procs, pluginID)
+ m.mu.Unlock()
+ if p != nil {
+ p.stop()
+ logger.Infof(ctx, "[plugin] host stopped %s", pluginID)
+ }
+ return nil
+}
+
+func (m *Manager) report(pluginID string, s State, err error) {
+ m.mu.Lock()
+ r := m.reporter
+ m.mu.Unlock()
+ if r == nil {
+ return
+ }
+ switch s {
+ case StateReady:
+ r.ReportRuntime(pluginID, true, nil)
+ case StateDegraded:
+ r.ReportRuntime(pluginID, false, err)
+ }
+}
+
+// Client returns a client for a running plugin. The error is a retryable
+// pluginapi unavailable error while the plugin is starting or restarting.
+func (m *Manager) Client(pluginID string) (*client.Client, error) {
+ m.mu.Lock()
+ p := m.procs[pluginID]
+ m.mu.Unlock()
+ if p == nil {
+ return nil, &pluginapi.Error{
+ Code: pluginapi.CodeUnavailable, Message: fmt.Sprintf("plugin %s is not running on this node", pluginID),
+ }
+ }
+ return p.Client()
+}
+
+// Close stops every plugin, for shutdown.
+func (m *Manager) Close() {
+ m.mu.Lock()
+ procs := m.procs
+ m.procs = map[string]*process{}
+ m.mu.Unlock()
+ var wg sync.WaitGroup
+ for _, p := range procs {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ p.stop()
+ }()
+ }
+ wg.Wait()
+}
diff --git a/internal/plugin/host/host_test.go b/internal/plugin/host/host_test.go
new file mode 100644
index 00000000..d62f1d7e
--- /dev/null
+++ b/internal/plugin/host/host_test.go
@@ -0,0 +1,297 @@
+package host
+
+import (
+ "context"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
+ "github.com/Tencent/WeKnora/internal/plugin/reconcile"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+var (
+ buildOnce sync.Once
+ binary string
+ buildErr error
+)
+
+// echoBinary builds testdata/echoplugin once per test run.
+func echoBinary(t *testing.T) string {
+ t.Helper()
+ buildOnce.Do(func() {
+ dir, err := os.MkdirTemp("", "echoplugin")
+ if err != nil {
+ buildErr = err
+ return
+ }
+ binary = filepath.Join(dir, "echo")
+ out, err := exec.Command("go", "build", "-o", binary, "./testdata/echoplugin").CombinedOutput()
+ if err != nil {
+ buildErr = err
+ t.Logf("%s", out)
+ }
+ })
+ if buildErr != nil {
+ t.Fatalf("build echo plugin: %v", buildErr)
+ }
+ return binary
+}
+
+// install lays out an extracted package with the echo binary.
+func install(t *testing.T, version string, fakeVersion string) *reconcile.Loaded {
+ t.Helper()
+ dir := t.TempDir()
+ bin := filepath.Join(dir, "bin", runtime.GOOS+"-"+runtime.GOARCH)
+ if err := os.MkdirAll(bin, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ data, err := os.ReadFile(echoBinary(t))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Extraction writes plain files; the host must make the entry runnable.
+ name := "echo"
+ if runtime.GOOS == "windows" {
+ name += ".exe"
+ }
+ if err := os.WriteFile(filepath.Join(bin, name), data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if fakeVersion != "" {
+ _ = os.WriteFile(filepath.Join(dir, "version"), []byte(fakeVersion), 0o644)
+ }
+ m := &manifest.Manifest{
+ SchemaVersion: manifest.SchemaVersion, ID: "acme.echo", Version: version, APIVersion: pluginapi.APIVersion,
+ Name: manifest.Text("Echo", nil), Publisher: manifest.Publisher{ID: "acme"},
+ Runtime: manifest.Runtime{Type: manifest.RuntimeHost, Kind: "binary", Entry: "bin/{os}-{arch}/echo"},
+ Contributes: manifest.Contributions{
+ manifest.PointWebSearch: {{ID: "echo", Name: manifest.Text("Echo", nil)}},
+ },
+ }
+ return &reconcile.Loaded{Manifest: m, Dir: dir}
+}
+
+type reports struct {
+ mu sync.Mutex
+ log []string
+}
+
+func (r *reports) ReportRuntime(id string, healthy bool, _ error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.log = append(r.log, id+":"+strconv.FormatBool(healthy))
+}
+
+func (r *reports) String() string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return strings.Join(r.log, ",")
+}
+
+func fastTimings(t *testing.T) {
+ t.Helper()
+ oldBackoff, oldInterval, oldGrace := restartBackoffFloor, healthInterval, stopGrace
+ restartBackoffFloor, healthInterval, stopGrace = 20*time.Millisecond, 100*time.Millisecond, 2*time.Second
+ t.Cleanup(func() { restartBackoffFloor, healthInterval, stopGrace = oldBackoff, oldInterval, oldGrace })
+}
+
+func search(t *testing.T, m *Manager, q string) (string, error) {
+ t.Helper()
+ c, err := m.Client("acme.echo")
+ if err != nil {
+ return "", err
+ }
+ var out pluginapi.SearchOutput
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err = c.Call(ctx, pluginapi.SearchPath("echo"), pluginapi.Envelope{}, pluginapi.SearchInput{Query: q}, &out)
+ if err != nil {
+ return "", err
+ }
+ return out.Results[0].Title, nil
+}
+
+func waitFor(t *testing.T, what string, cond func() bool) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for !cond() {
+ if time.Now().After(deadline) {
+ t.Fatalf("timed out waiting for %s", what)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
+
+func TestHostRunsRestartsAndStopsAPlugin(t *testing.T) {
+ fastTimings(t)
+ t.Setenv("DB_PASSWORD", "must-not-leak")
+ ctx := context.Background()
+ m := NewManager()
+ rep := &reports{}
+ m.SetReporter(rep)
+ defer m.Close()
+
+ if err := m.Activate(ctx, install(t, "1.0.0", "")); err != nil {
+ t.Fatalf("Activate: %v", err)
+ }
+ if got, err := search(t, m, "hello"); err != nil || got != "hello" {
+ t.Fatalf("search = %q, %v", got, err)
+ }
+ env, err := search(t, m, "env")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(env, "DB_PASSWORD") || !strings.Contains(env, "HTTPS_PROXY") ||
+ !strings.Contains(env, pluginapi.EnvToken) {
+ t.Fatalf("plugin environment = %s", env)
+ }
+
+ pid1, _ := search(t, m, "pid")
+ _, _ = search(t, m, "crash")
+ waitFor(t, "restart", func() bool {
+ pid, err := search(t, m, "pid")
+ return err == nil && pid != pid1
+ })
+ if !strings.Contains(rep.String(), "acme.echo:false") || !strings.HasSuffix(rep.String(), "acme.echo:true") {
+ t.Fatalf("reports = %s", rep)
+ }
+
+ pid2, _ := search(t, m, "pid")
+ if err := m.Deactivate(ctx, "acme.echo"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Client("acme.echo"); err == nil {
+ t.Fatal("a stopped plugin must have no client")
+ }
+ if runtime.GOOS != "windows" {
+ n, _ := strconv.Atoi(pid2)
+ if proc, err := os.FindProcess(n); err == nil && proc.Signal(syscall.Signal(0)) == nil {
+ t.Fatalf("process %d still runs after Deactivate", n)
+ }
+ }
+}
+
+func TestHostRefusesAProcessThatIsNotThePackage(t *testing.T) {
+ fastTimings(t)
+ m := NewManager()
+ defer m.Close()
+ err := m.Activate(context.Background(), install(t, "1.0.0", "9.9.9"))
+ if err == nil || !strings.Contains(err.Error(), "reports acme.echo@9.9.9") {
+ t.Fatalf("want a manifest mismatch, got %v", err)
+ }
+ l := install(t, "1.0.0", "")
+ l.Manifest.Contributes[manifest.PointWebSearch] = append(l.Manifest.Contributes[manifest.PointWebSearch],
+ manifest.Contribution{ID: "missing", Name: manifest.Text("Missing", nil)})
+ if err := m.Activate(context.Background(), l); err == nil || !strings.Contains(err.Error(), "webSearch/missing") {
+ t.Fatalf("want a missing contribution error, got %v", err)
+ }
+ l = install(t, "1.0.0", "")
+ l.Manifest.Runtime.Entry = "bin/{os}-{arch}/nope"
+ if err := m.Activate(context.Background(), l); err == nil || !strings.Contains(err.Error(), "has no") {
+ t.Fatalf("want a missing build error, got %v", err)
+ }
+}
+
+func TestUpgradeSwapsProcessesInPlace(t *testing.T) {
+ fastTimings(t)
+ ctx := context.Background()
+ m := NewManager()
+ defer m.Close()
+ if err := m.Activate(ctx, install(t, "1.0.0", "")); err != nil {
+ t.Fatal(err)
+ }
+ pid1, _ := search(t, m, "pid")
+ if err := m.Activate(ctx, install(t, "1.1.0", "")); err != nil {
+ t.Fatal(err)
+ }
+ pid2, err := search(t, m, "pid")
+ if err != nil || pid2 == pid1 {
+ t.Fatalf("after upgrade pid = %s (was %s), %v", pid2, pid1, err)
+ }
+}
+
+func TestEgressProxyEnforcesThePolicy(t *testing.T) {
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte("hello from " + r.Host))
+ }))
+ defer upstream.Close()
+ p, err := startEgressProxy("acme.echo", []string{"api.allowed.test", "*.wild.test"}, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer p.Close()
+ // Every name resolves to the test server; the policy is what differs.
+ p.dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, network, upstream.Listener.Addr().String())
+ }
+ proxyURL, _ := url.Parse(p.URL())
+ c := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
+ for host, want := range map[string]int{
+ "api.allowed.test": 200, "x.wild.test": 200, "wild.test": 403, "evil.test": 403,
+ } {
+ resp, err := c.Get("http://" + host + "/")
+ if err != nil {
+ t.Fatalf("%s: %v", host, err)
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != want {
+ t.Errorf("%s: status %d, want %d", host, resp.StatusCode, want)
+ }
+ }
+}
+
+func TestEgressProxyTunnelsHTTPS(t *testing.T) {
+ upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte("secure"))
+ }))
+ defer upstream.Close()
+ p, err := startEgressProxy("acme.echo", []string{"*.wild.test"}, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer p.Close()
+ p.dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, network, upstream.Listener.Addr().String())
+ }
+ proxyURL, _ := url.Parse(p.URL())
+ tr := upstream.Client().Transport.(*http.Transport).Clone()
+ tr.Proxy = http.ProxyURL(proxyURL)
+ tr.TLSClientConfig.InsecureSkipVerify = true
+ c := &http.Client{Transport: tr}
+
+ resp, err := c.Get("https://api.wild.test/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ body := make([]byte, 16)
+ n, _ := resp.Body.Read(body)
+ _ = resp.Body.Close()
+ if string(body[:n]) != "secure" {
+ t.Fatalf("tunnelled body = %q", body[:n])
+ }
+ if _, err := c.Get("https://evil.test/"); err == nil || !strings.Contains(err.Error(), "Forbidden") {
+ t.Fatalf("CONNECT to a host outside the policy must be refused, got %v", err)
+ }
+}
+
+func TestProtocolVersionsAgree(t *testing.T) {
+ if manifest.ExtensionAPIVersion != pluginapi.APIVersion {
+ t.Fatalf("manifest accepts %q, the protocol package speaks %q",
+ manifest.ExtensionAPIVersion, pluginapi.APIVersion)
+ }
+}
diff --git a/internal/plugin/host/proc_linux.go b/internal/plugin/host/proc_linux.go
new file mode 100644
index 00000000..1529fc10
--- /dev/null
+++ b/internal/plugin/host/proc_linux.go
@@ -0,0 +1,16 @@
+//go:build linux
+
+package host
+
+import (
+ "os/exec"
+ "syscall"
+)
+
+// configureChild puts the plugin in its own process group and has the kernel
+// kill it if WeKnora dies, so a crash never leaves orphaned plugins.
+func configureChild(cmd *exec.Cmd) {
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
+}
+
+func terminate(cmd *exec.Cmd) error { return cmd.Process.Signal(syscall.SIGTERM) }
diff --git a/internal/plugin/host/proc_unix.go b/internal/plugin/host/proc_unix.go
new file mode 100644
index 00000000..4007b020
--- /dev/null
+++ b/internal/plugin/host/proc_unix.go
@@ -0,0 +1,16 @@
+//go:build unix && !linux
+
+package host
+
+import (
+ "os/exec"
+ "syscall"
+)
+
+// configureChild puts the plugin in its own process group, so a signal
+// meant for WeKnora's terminal does not reach it directly.
+func configureChild(cmd *exec.Cmd) {
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+}
+
+func terminate(cmd *exec.Cmd) error { return cmd.Process.Signal(syscall.SIGTERM) }
diff --git a/internal/plugin/host/proc_windows.go b/internal/plugin/host/proc_windows.go
new file mode 100644
index 00000000..903de6b0
--- /dev/null
+++ b/internal/plugin/host/proc_windows.go
@@ -0,0 +1,11 @@
+//go:build windows
+
+package host
+
+import "os/exec"
+
+func configureChild(*exec.Cmd) {}
+
+// terminate kills the plugin: Windows has no SIGTERM for console-less
+// children.
+func terminate(cmd *exec.Cmd) error { return cmd.Process.Kill() }
diff --git a/internal/plugin/host/process.go b/internal/plugin/host/process.go
new file mode 100644
index 00000000..e40eb28f
--- /dev/null
+++ b/internal/plugin/host/process.go
@@ -0,0 +1,438 @@
+package host
+
+import (
+ "bufio"
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
+ "github.com/Tencent/WeKnora/internal/utils"
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// Timing of a host plugin's life. Variables so tests can shrink them.
+var (
+ handshakeTimeout = 30 * time.Second
+ healthInterval = 15 * time.Second
+ healthTimeout = 5 * time.Second
+ unhealthyLimit = 3
+ stopGrace = 10 * time.Second
+ restartBackoffFloor = time.Second
+ restartBackoffCeiling = time.Minute
+)
+
+// State of a supervised process.
+type State string
+
+// Process states reported to the reconciler.
+const (
+ StateStarting State = "starting"
+ StateReady State = "ready"
+ StateDegraded State = "degraded"
+ StateStopped State = "stopped"
+)
+
+// spec is everything needed to start one plugin version.
+type spec struct {
+ m *manifest.Manifest
+ dir string // extracted package
+}
+
+// entryPath resolves runtime.entry for this machine inside the package.
+func entryPath(m *manifest.Manifest, dir string) (string, error) {
+ rel := strings.NewReplacer("{os}", runtime.GOOS, "{arch}", runtime.GOARCH).Replace(m.Runtime.Entry)
+ if runtime.GOOS == "windows" && filepath.Ext(rel) == "" {
+ rel += ".exe"
+ }
+ p, err := utils.SafeJoinUnderBase(dir, rel)
+ if err != nil {
+ return "", fmt.Errorf("runtime.entry %q: %w", m.Runtime.Entry, err)
+ }
+ info, err := os.Stat(p)
+ if err != nil {
+ return "", fmt.Errorf("the package has no %s build (%s)", runtime.GOOS+"/"+runtime.GOARCH, rel)
+ }
+ if !info.Mode().IsRegular() {
+ return "", fmt.Errorf("runtime.entry %s is not a file", rel)
+ }
+ // Packages are extracted without modes; the entry has to be executable.
+ if err := os.Chmod(p, 0o755); err != nil {
+ return "", err
+ }
+ return p, nil
+}
+
+// process is one running plugin version and its supervisor.
+type process struct {
+ spec spec
+ onState func(State, error)
+
+ mu sync.RWMutex
+ client *client.Client
+ state State
+ lastErr error
+
+ cancel context.CancelFunc
+ done chan struct{}
+ proxy *egressProxy
+ sockDir string
+}
+
+// launched is one started child.
+type launched struct {
+ cmd *exec.Cmd
+ client *client.Client
+ exited chan error
+}
+
+// startProcess starts the plugin, waits until it is ready and keeps it
+// running until stop. It returns once the first start succeeded or failed.
+func startProcess(sp spec, onState func(State, error)) (*process, error) {
+ entry, err := entryPath(sp.m, sp.dir)
+ if err != nil {
+ return nil, err
+ }
+ sockDir, err := socketDir()
+ if err != nil {
+ return nil, err
+ }
+ proxy, err := startEgressProxy(sp.m.ID, sp.m.Permissions.Egress, func(f string, a ...any) {
+ logger.Warnf(context.Background(), f, a...)
+ })
+ if err != nil {
+ _ = os.RemoveAll(sockDir)
+ return nil, err
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ p := &process{spec: sp, onState: onState, cancel: cancel, done: make(chan struct{}), proxy: proxy, sockDir: sockDir}
+ p.setState(StateStarting, nil)
+ first, err := p.launch(ctx, entry)
+ if err != nil {
+ cancel()
+ proxy.Close()
+ _ = os.RemoveAll(sockDir)
+ close(p.done)
+ p.setState(StateStopped, err)
+ return nil, err
+ }
+ // Ready before Activate returns: callers route to the plugin right away.
+ p.setClient(first.client)
+ p.setState(StateReady, nil)
+ go p.supervise(ctx, entry, first)
+ return p, nil
+}
+
+// socketDir makes a short private directory for the socket: unix socket
+// paths are limited to about 100 bytes.
+func socketDir() (string, error) {
+ base := os.Getenv("WEKNORA_PLUGIN_SOCKET_DIR")
+ if base == "" {
+ base = os.TempDir()
+ }
+ return os.MkdirTemp(base, "wkp")
+}
+
+func randomToken() string {
+ b := make([]byte, 32)
+ _, _ = rand.Read(b)
+ return hex.EncodeToString(b)
+}
+
+// childEnv is the plugin's whole environment. It is built from scratch so
+// WeKnora's own secrets (database, AES key, vendor keys) never reach plugin
+// code.
+func (p *process) childEnv(socket, token string) []string {
+ env := []string{
+ pluginapi.EnvSocket + "=" + socket,
+ pluginapi.EnvNetwork + "=unix",
+ pluginapi.EnvToken + "=" + token,
+ pluginapi.EnvPluginID + "=" + p.spec.m.ID,
+ pluginapi.EnvPluginVersion + "=" + p.spec.m.Version,
+ "HOME=" + p.sockDir,
+ "TMPDIR=" + p.sockDir,
+ "HTTP_PROXY=" + p.proxy.URL(),
+ "HTTPS_PROXY=" + p.proxy.URL(),
+ "http_proxy=" + p.proxy.URL(),
+ "https_proxy=" + p.proxy.URL(),
+ "NO_PROXY=",
+ "no_proxy=",
+ }
+ for _, k := range []string{"PATH", "LANG", "LC_ALL", "TZ", "SYSTEMROOT", "WINDIR"} {
+ if v, ok := os.LookupEnv(k); ok {
+ env = append(env, k+"="+v)
+ }
+ }
+ return env
+}
+
+// launch starts the child and waits for handshake, health and a manifest
+// that matches the installed package.
+func (p *process) launch(ctx context.Context, entry string) (*launched, error) {
+ socket := filepath.Join(p.sockDir, "p.sock")
+ _ = os.Remove(socket)
+ token := randomToken()
+ cmd := exec.Command(entry)
+ cmd.Dir = p.spec.dir
+ cmd.Env = p.childEnv(socket, token)
+ configureChild(cmd)
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return nil, err
+ }
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return nil, err
+ }
+ if err := cmd.Start(); err != nil {
+ return nil, fmt.Errorf("start %s: %w", filepath.Base(entry), err)
+ }
+ id := p.spec.m.ID + "@" + p.spec.m.Version
+ handshake := make(chan pluginapi.Handshake, 1)
+ hsErr := make(chan error, 1)
+ go forwardLogs(stderr, id, nil, nil)
+ go forwardLogs(stdout, id, handshake, hsErr)
+ exited := make(chan error, 1)
+ go func() { exited <- cmd.Wait() }()
+
+ kill := func() {
+ stopChild(cmd, exited)
+ }
+ timer := time.NewTimer(handshakeTimeout)
+ defer timer.Stop()
+ var hs pluginapi.Handshake
+ select {
+ case hs = <-handshake:
+ case err := <-hsErr:
+ kill()
+ return nil, err
+ case err := <-exited:
+ exited <- err // keep it for supervise
+ return nil, fmt.Errorf("plugin exited before it was ready: %v", err)
+ case <-timer.C:
+ kill()
+ return nil, fmt.Errorf("plugin did not print its handshake within %s", handshakeTimeout)
+ case <-ctx.Done():
+ kill()
+ return nil, ctx.Err()
+ }
+ if hs.Network == "unix" && hs.Address != socket {
+ kill()
+ return nil, fmt.Errorf("plugin listens on %s, not the socket it was given", hs.Address)
+ }
+ c := client.ForHandshake(hs, client.Bearer(token))
+ cctx, cancel := context.WithTimeout(ctx, healthTimeout)
+ defer cancel()
+ if err := c.Health(cctx); err != nil {
+ c.Close()
+ kill()
+ return nil, fmt.Errorf("plugin is not healthy: %w", err)
+ }
+ m, err := c.Manifest(cctx)
+ if err != nil {
+ c.Close()
+ kill()
+ return nil, fmt.Errorf("read plugin manifest: %w", err)
+ }
+ if err := checkServedManifest(p.spec.m, m); err != nil {
+ c.Close()
+ kill()
+ return nil, err
+ }
+ return &launched{cmd: cmd, client: c, exited: exited}, nil
+}
+
+// checkServedManifest makes sure the process is the package that was
+// installed and serves what the manifest promised.
+func checkServedManifest(want *manifest.Manifest, got *pluginapi.Manifest) error {
+ if got.ID != want.ID || got.Version != want.Version {
+ return fmt.Errorf("process reports %s@%s, the package is %s@%s", got.ID, got.Version, want.ID, want.Version)
+ }
+ if got.APIVersion != pluginapi.APIVersion {
+ return fmt.Errorf("process speaks %q, this WeKnora speaks %q", got.APIVersion, pluginapi.APIVersion)
+ }
+ for point, contribs := range want.Contributes {
+ if info, ok := manifest.LookupPoint(point); !ok || info.Declarative {
+ continue
+ }
+ served := map[string]bool{}
+ for _, id := range got.Contributes[string(point)] {
+ served[id] = true
+ }
+ for _, c := range contribs {
+ if !served[c.ID] {
+ return fmt.Errorf("plugin.yaml declares %s/%s but the process does not serve it", point, c.ID)
+ }
+ }
+ }
+ return nil
+}
+
+// forwardLogs copies a child's output into WeKnora's logs. On stdout it
+// also watches for the handshake line.
+func forwardLogs(r io.Reader, id string, handshake chan<- pluginapi.Handshake, hsErr chan<- error) {
+ sc := bufio.NewScanner(r)
+ sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
+ seen := false
+ for sc.Scan() {
+ line := sc.Text()
+ if handshake != nil && !seen {
+ hs, ok, err := pluginapi.ParseHandshake(line)
+ if ok {
+ seen = true
+ if err != nil {
+ hsErr <- err
+ } else {
+ handshake <- hs
+ }
+ continue
+ }
+ }
+ logger.Infof(context.Background(), "[plugin %s] %s", id, line)
+ }
+}
+
+func (p *process) setState(s State, err error) {
+ p.mu.Lock()
+ changed := p.state != s || (err != nil) != (p.lastErr != nil)
+ p.state, p.lastErr = s, err
+ p.mu.Unlock()
+ if changed && p.onState != nil {
+ p.onState(s, err)
+ }
+}
+
+func (p *process) setClient(c *client.Client) {
+ p.mu.Lock()
+ old := p.client
+ p.client = c
+ p.mu.Unlock()
+ if old != nil && old != c {
+ old.Close()
+ }
+}
+
+// Client returns the client of the running child, or an unavailable error.
+func (p *process) Client() (*client.Client, error) {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+ if p.client == nil || p.state != StateReady {
+ msg := fmt.Sprintf("plugin %s is %s", p.spec.m.ID, p.state)
+ if p.lastErr != nil {
+ msg += ": " + p.lastErr.Error()
+ }
+ return nil, &pluginapi.Error{Code: pluginapi.CodeUnavailable, Message: msg, Retryable: true}
+ }
+ return p.client, nil
+}
+
+// supervise keeps the plugin running: health checks, restart with backoff
+// after a crash or repeated failed checks.
+func (p *process) supervise(ctx context.Context, entry string, cur *launched) {
+ defer close(p.done)
+ backoff := restartBackoffFloor
+ for {
+ p.setClient(cur.client)
+ p.setState(StateReady, nil)
+ exitErr := p.watch(ctx, cur)
+ p.setClient(nil)
+ if ctx.Err() != nil {
+ p.setState(StateStopped, nil)
+ return
+ }
+ logger.Warnf(ctx, "[plugin] %s stopped unexpectedly: %v; restarting in %s", p.spec.m.ID, exitErr, backoff)
+ p.setState(StateDegraded, exitErr)
+ for {
+ select {
+ case <-ctx.Done():
+ p.setState(StateStopped, nil)
+ return
+ case <-time.After(backoff):
+ }
+ backoff = min(backoff*2, restartBackoffCeiling)
+ next, err := p.launch(ctx, entry)
+ if err == nil {
+ cur = next
+ backoff = restartBackoffFloor
+ logger.Infof(ctx, "[plugin] %s restarted", p.spec.m.ID)
+ break
+ }
+ if ctx.Err() != nil {
+ p.setState(StateStopped, nil)
+ return
+ }
+ logger.Warnf(ctx, "[plugin] %s restart failed: %v; next try in %s", p.spec.m.ID, err, backoff)
+ p.setState(StateDegraded, err)
+ }
+ }
+}
+
+// watch returns when the child exits, fails its health checks, or ctx ends
+// (then it stops the child).
+func (p *process) watch(ctx context.Context, cur *launched) error {
+ t := time.NewTicker(healthInterval)
+ defer t.Stop()
+ failures := 0
+ for {
+ select {
+ case err := <-cur.exited:
+ if err == nil {
+ err = errors.New("exited")
+ }
+ return err
+ case <-ctx.Done():
+ stopChild(cur.cmd, cur.exited)
+ return nil
+ case <-t.C:
+ hctx, cancel := context.WithTimeout(ctx, healthTimeout)
+ err := cur.client.Health(hctx)
+ cancel()
+ if err == nil {
+ failures = 0
+ continue
+ }
+ failures++
+ if failures >= unhealthyLimit {
+ stopChild(cur.cmd, cur.exited)
+ return fmt.Errorf("failed %d health checks: %w", failures, err)
+ }
+ }
+ }
+}
+
+// stop ends the plugin and waits for it to exit.
+func (p *process) stop() {
+ p.cancel()
+ <-p.done
+ p.proxy.Close()
+ _ = os.RemoveAll(p.sockDir)
+}
+
+// stopChild asks the child to exit, then kills it after stopGrace. exited
+// is the channel its Wait result arrives on.
+func stopChild(cmd *exec.Cmd, exited chan error) {
+ if cmd.Process == nil {
+ return
+ }
+ _ = terminate(cmd)
+ select {
+ case err := <-exited:
+ exited <- err
+ case <-time.After(stopGrace):
+ _ = cmd.Process.Kill()
+ err := <-exited
+ exited <- err
+ }
+}
diff --git a/internal/plugin/host/proxy.go b/internal/plugin/host/proxy.go
new file mode 100644
index 00000000..d5adb250
--- /dev/null
+++ b/internal/plugin/host/proxy.go
@@ -0,0 +1,192 @@
+package host
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/utils"
+)
+
+// egressPolicy decides which hosts a plugin may reach: the manifest's
+// permissions.egress patterns ("api.example.com", "*.atlassian.net"), which
+// the system administrator accepted at install time.
+type egressPolicy struct {
+ any bool // "*": every public host
+ exact map[string]bool
+ suffixes []string // ".atlassian.net"
+}
+
+func newEgressPolicy(patterns []string) egressPolicy {
+ p := egressPolicy{exact: map[string]bool{}}
+ for _, raw := range patterns {
+ pat := strings.ToLower(strings.TrimSpace(raw))
+ if pat == "" {
+ continue
+ }
+ if pat == "*" {
+ p.any = true
+ continue
+ }
+ if rest, ok := strings.CutPrefix(pat, "*."); ok {
+ p.suffixes = append(p.suffixes, "."+rest)
+ continue
+ }
+ p.exact[pat] = true
+ }
+ return p
+}
+
+func (p egressPolicy) allows(host string) bool {
+ if p.any {
+ // The SSRF-safe dialer still refuses private addresses.
+ return true
+ }
+ host = strings.ToLower(strings.TrimSuffix(host, "."))
+ if p.exact[host] {
+ return true
+ }
+ for _, s := range p.suffixes {
+ if strings.HasSuffix(host, s) {
+ return true
+ }
+ }
+ return false
+}
+
+// egressProxy is the HTTP proxy a host plugin's traffic goes through
+// (HTTP_PROXY / HTTPS_PROXY in its environment). It forwards only to hosts
+// the plugin was granted and dials through the SSRF-safe dialer, so a
+// granted name that resolves to a private address is still refused unless
+// the operator whitelisted it. A plugin can ignore the proxy variables; on
+// Linux a network namespace can make the proxy the only way out.
+type egressProxy struct {
+ pluginID string
+ policy egressPolicy
+ ln net.Listener
+ srv *http.Server
+ dial func(ctx context.Context, network, addr string) (net.Conn, error)
+ logf func(format string, args ...any)
+ wg sync.WaitGroup
+}
+
+func startEgressProxy(pluginID string, patterns []string, logf func(string, ...any)) (*egressProxy, error) {
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ return nil, fmt.Errorf("start egress proxy: %w", err)
+ }
+ p := &egressProxy{
+ pluginID: pluginID, policy: newEgressPolicy(patterns), ln: ln,
+ dial: utils.SSRFSafeDialContext, logf: logf,
+ }
+ p.srv = &http.Server{Handler: p, ReadHeaderTimeout: 30 * time.Second}
+ p.wg.Add(1)
+ go func() {
+ defer p.wg.Done()
+ _ = p.srv.Serve(ln)
+ }()
+ return p, nil
+}
+
+// URL is the proxy address for the plugin's environment.
+func (p *egressProxy) URL() string { return "http://" + p.ln.Addr().String() }
+
+func (p *egressProxy) Close() {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ _ = p.srv.Shutdown(ctx)
+ p.wg.Wait()
+}
+
+func (p *egressProxy) deny(w http.ResponseWriter, host string) {
+ p.logf("[plugin] %s: egress to %s refused (not in permissions.egress)", p.pluginID, host)
+ http.Error(w, fmt.Sprintf("egress to %s is not permitted for plugin %s", host, p.pluginID), http.StatusForbidden)
+}
+
+func (p *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodConnect {
+ p.tunnel(w, r)
+ return
+ }
+ if r.URL.Host == "" {
+ http.Error(w, "not a proxy request", http.StatusBadRequest)
+ return
+ }
+ host := r.URL.Hostname()
+ if !p.policy.allows(host) {
+ p.deny(w, host)
+ return
+ }
+ out := r.Clone(r.Context())
+ out.RequestURI = ""
+ out.Header.Del("Proxy-Connection")
+ out.Header.Del("Proxy-Authorization")
+ tr := &http.Transport{DialContext: p.dial, ResponseHeaderTimeout: 2 * time.Minute}
+ defer tr.CloseIdleConnections()
+ resp, err := tr.RoundTrip(out)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ defer func() { _ = resp.Body.Close() }()
+ for k, vs := range resp.Header {
+ for _, v := range vs {
+ w.Header().Add(k, v)
+ }
+ }
+ w.WriteHeader(resp.StatusCode)
+ _, _ = io.Copy(w, resp.Body)
+}
+
+// tunnel handles CONNECT, which HTTPS traffic uses: the proxy sees only the
+// host and port, which is what the policy needs.
+func (p *egressProxy) tunnel(w http.ResponseWriter, r *http.Request) {
+ host, _, err := net.SplitHostPort(r.Host)
+ if err != nil {
+ http.Error(w, "bad CONNECT target", http.StatusBadRequest)
+ return
+ }
+ if !p.policy.allows(host) {
+ p.deny(w, host)
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+ upstream, err := p.dial(ctx, "tcp", r.Host)
+ cancel()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ _ = upstream.Close()
+ http.Error(w, "hijacking unsupported", http.StatusInternalServerError)
+ return
+ }
+ client, buf, err := hj.Hijack()
+ if err != nil {
+ _ = upstream.Close()
+ return
+ }
+ if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
+ _ = upstream.Close()
+ _ = client.Close()
+ return
+ }
+ if buf.Reader.Buffered() > 0 {
+ pending := make([]byte, buf.Reader.Buffered())
+ _, _ = buf.Read(pending)
+ _, _ = upstream.Write(pending)
+ }
+ go func() {
+ _, _ = io.Copy(upstream, client)
+ _ = upstream.Close()
+ }()
+ _, _ = io.Copy(client, upstream)
+ _ = client.Close()
+}
diff --git a/internal/plugin/host/testdata/echoplugin/main.go b/internal/plugin/host/testdata/echoplugin/main.go
new file mode 100644
index 00000000..c3e3d3ec
--- /dev/null
+++ b/internal/plugin/host/testdata/echoplugin/main.go
@@ -0,0 +1,44 @@
+// Command echoplugin is a host plugin for the host package's tests. Its
+// behaviour is driven by the query it is asked and by a "version" file next
+// to it, so a test can install a package that lies about its version.
+package main
+
+import (
+ "context"
+ "log"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/Tencent/WeKnora/pluginsdk"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+func main() {
+ version := os.Getenv(pluginapi.EnvPluginVersion)
+ if b, err := os.ReadFile("version"); err == nil {
+ version = strings.TrimSpace(string(b))
+ }
+ p := pluginsdk.New(pluginsdk.Info{ID: os.Getenv(pluginapi.EnvPluginID), Version: version})
+ p.WebSearch("echo", pluginsdk.WebSearchFunc(func(_ context.Context, _ *pluginsdk.Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
+ switch in.Query {
+ case "crash":
+ os.Exit(3)
+ case "env":
+ var keys []string
+ for _, kv := range os.Environ() {
+ k, _, _ := strings.Cut(kv, "=")
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: strings.Join(keys, ","), URL: "env"}}}, nil
+ case "pid":
+ return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: strconv.Itoa(os.Getpid()), URL: "pid"}}}, nil
+ }
+ return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{Title: in.Query, URL: "echo"}}}, nil
+ }))
+ if err := p.Serve(); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/internal/plugin/install/install.go b/internal/plugin/install/install.go
index 009280b9..70b99239 100644
--- a/internal/plugin/install/install.go
+++ b/internal/plugin/install/install.go
@@ -12,7 +12,10 @@ import (
"io"
"net/http"
"net/url"
+ "path"
+ goruntime "runtime"
"sort"
+ "strings"
"time"
"golang.org/x/mod/semver"
@@ -41,8 +44,29 @@ func invalid(format string, args ...any) error {
}
// supportedRuntimes are the runtimes an installed package may use today.
-// Code plugins arrive with the plugin host.
-var supportedRuntimes = map[manifest.RuntimeType]bool{manifest.RuntimeDeclarative: true}
+var supportedRuntimes = map[manifest.RuntimeType]bool{
+ manifest.RuntimeDeclarative: true,
+ manifest.RuntimeHost: true,
+}
+
+// checkHostRuntime makes sure this server can run a host plugin: a binary
+// built for its OS and architecture. Other kinds need a host image that
+// carries their interpreter.
+func checkHostRuntime(p *pkg.Package) error {
+ rt := p.Manifest.Runtime
+ if rt.Kind != "binary" {
+ return invalid("runtime.kind %q is not supported yet; host plugins must be binaries", rt.Kind)
+ }
+ entry := strings.NewReplacer("{os}", goruntime.GOOS, "{arch}", goruntime.GOARCH).Replace(rt.Entry)
+ if goruntime.GOOS == "windows" && path.Ext(entry) == "" {
+ entry += ".exe"
+ }
+ if _, ok := p.ReadFile(entry); !ok {
+ return invalid("the package has no build for this server (%s/%s): %s is missing",
+ goruntime.GOOS, goruntime.GOARCH, entry)
+ }
+ return nil
+}
// Syncer is the node's reconciler as the installer uses it.
type Syncer interface {
@@ -152,9 +176,14 @@ func (s *Service) open(data []byte) (*pkg.Package, error) {
return nil, &InvalidError{Err: err}
}
if !supportedRuntimes[p.Manifest.Runtime.Type] {
- return nil, invalid("runtime %q is not supported yet; only declarative plugins can be installed",
+ return nil, invalid("runtime %q is not supported yet; declarative and host plugins can be installed",
p.Manifest.Runtime.Type)
}
+ if p.Manifest.Runtime.Type == manifest.RuntimeHost {
+ if err := checkHostRuntime(p); err != nil {
+ return nil, err
+ }
+ }
if err := p.Manifest.CheckEngines(s.hostVersion); err != nil {
return nil, &InvalidError{Err: err}
}
diff --git a/internal/plugin/install/install_test.go b/internal/plugin/install/install_test.go
index 5b3fbdd1..88205325 100644
--- a/internal/plugin/install/install_test.go
+++ b/internal/plugin/install/install_test.go
@@ -3,6 +3,7 @@ package install
import (
"context"
"errors"
+ goruntime "runtime"
"strings"
"testing"
@@ -127,3 +128,36 @@ func isInvalid(err error) bool {
var ie *InvalidError
return errors.As(err, &ie)
}
+
+func hostPackage(t *testing.T, kind, binPath string) []byte {
+ return plugintest.Zip(t, map[string]string{
+ "plugin.yaml": "schemaVersion: 1\nid: acme.search\nversion: 1.0.0\napiVersion: weknora.plugin/v1\n" +
+ "name: { en-US: ACME Search }\npublisher: { id: acme }\n" +
+ "runtime: { type: host, kind: " + kind + ", entry: \"bin/{os}-{arch}/search\" }\n" +
+ "permissions: { egress: [api.acme.example] }\n" +
+ "contributes:\n webSearch:\n - { id: search, name: ACME Search }\n",
+ binPath: "binary",
+ })
+}
+
+func TestInstallHostPlugins(t *testing.T) {
+ ctx := context.Background()
+ s, _, _, _ := newService(t)
+ name := "search"
+ if goruntime.GOOS == "windows" {
+ name += ".exe"
+ }
+ here := "bin/" + goruntime.GOOS + "-" + goruntime.GOARCH + "/" + name
+ p, err := s.Inspect(ctx, hostPackage(t, "binary", here))
+ if err != nil || p.Manifest.Permissions.Egress[0] != "api.acme.example" {
+ t.Fatalf("Inspect = %+v, %v", p, err)
+ }
+ if _, err := s.Inspect(ctx, hostPackage(t, "binary", "bin/plan9-mips/search")); !isInvalid(err) ||
+ !strings.Contains(err.Error(), "no build for this server") {
+ t.Fatalf("want a missing build error, got %v", err)
+ }
+ if _, err := s.Inspect(ctx, hostPackage(t, "python", here)); !isInvalid(err) ||
+ !strings.Contains(err.Error(), "must be binaries") {
+ t.Fatalf("want an unsupported kind error, got %v", err)
+ }
+}
diff --git a/internal/plugin/manifest/manifest.go b/internal/plugin/manifest/manifest.go
index bb43e4aa..cc5f3fec 100644
--- a/internal/plugin/manifest/manifest.go
+++ b/internal/plugin/manifest/manifest.go
@@ -57,15 +57,15 @@ type Manifest struct {
// Builtin marks a plugin compiled into WeKnora. It cannot be set from a
// manifest file.
- Builtin bool `json:"builtin,omitempty" yaml:"-"`
+ Builtin bool `json:"builtin,omitempty" yaml:"-"`
// Required marks a builtin a tenant may not disable (the core agent
// tools, the default parser).
Required bool `json:"required,omitempty" yaml:"-"`
- Runtime Runtime `json:"runtime" yaml:"runtime"`
- Permissions Permissions `json:"permissions,omitzero" yaml:"permissions"`
- Config ConfigSchemas `json:"config,omitzero" yaml:"config"`
- Contributes Contributions `json:"contributes" yaml:"contributes"`
+ Runtime Runtime `json:"runtime" yaml:"runtime"`
+ Permissions Permissions `json:"permissions,omitzero" yaml:"permissions"`
+ Config ConfigSchemas `json:"config,omitzero" yaml:"config"`
+ Contributes Contributions `json:"contributes" yaml:"contributes"`
}
// Engines constrains which WeKnora versions a plugin runs on.
@@ -84,12 +84,12 @@ type Publisher struct {
// Runtime says how a plugin's code runs.
type Runtime struct {
- Type RuntimeType `json:"type" yaml:"type"`
+ Type RuntimeType `json:"type" yaml:"type"`
// Kind is the process runtime for host plugins: binary, python or node.
- Kind string `json:"kind,omitempty" yaml:"kind"`
+ Kind string `json:"kind,omitempty" yaml:"kind"`
// Entry is the command inside the package, with {os} and {arch}
// placeholders for binaries.
- Entry string `json:"entry,omitempty" yaml:"entry"`
+ Entry string `json:"entry,omitempty" yaml:"entry"`
// Singleton allows only one live instance across the cluster (long-lived
// connections such as an IM WebSocket).
Singleton bool `json:"singleton,omitempty" yaml:"singleton"`
@@ -106,17 +106,17 @@ type Resources struct {
// install time.
type Permissions struct {
// Egress lists host patterns the plugin may reach ("*.atlassian.net").
- Egress []string `json:"egress,omitempty" yaml:"egress"`
+ Egress []string `json:"egress,omitempty" yaml:"egress"`
// HostAPI lists WeKnora API scopes the plugin may call back with.
HostAPI []string `json:"hostApi,omitempty" yaml:"hostApi"`
// Events lists asynchronous events the plugin subscribes to.
- Events []string `json:"events,omitempty" yaml:"events"`
+ Events []string `json:"events,omitempty" yaml:"events"`
}
// ConfigSchemas points at the JSON Schemas of plugin-level configuration.
type ConfigSchemas struct {
- System string `json:"system,omitempty" yaml:"system"`
- Tenant string `json:"tenant,omitempty" yaml:"tenant"`
+ System string `json:"system,omitempty" yaml:"system"`
+ Tenant string `json:"tenant,omitempty" yaml:"tenant"`
// SystemSchema and TenantSchema are the schema files' contents as JSON,
// filled in when a package is opened.
SystemSchema json.RawMessage `json:"systemSchema,omitempty" yaml:"-"`
@@ -130,40 +130,40 @@ type Contributions map[Point][]Contribution
// such as one connector.
type Contribution struct {
// ID is local to the plugin; QualifiedID adds the plugin ID.
- ID string `json:"id" yaml:"id"`
- Name LocalizedText `json:"name" yaml:"name"`
- Description LocalizedText `json:"description,omitzero" yaml:"description"`
- Icon string `json:"icon,omitempty" yaml:"icon"`
+ ID string `json:"id" yaml:"id"`
+ Name LocalizedText `json:"name" yaml:"name"`
+ Description LocalizedText `json:"description,omitzero" yaml:"description"`
+ Icon string `json:"icon,omitempty" yaml:"icon"`
// Aliases are the short names builtins were stored under before plugins
// existed ("feishu"). Rows keep them, so lookups must keep resolving
// them. Only builtins may declare aliases.
- Aliases []string `json:"aliases,omitempty" yaml:"-"`
- Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities"`
+ Aliases []string `json:"aliases,omitempty" yaml:"-"`
+ Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities"`
// Order sorts contributions of one point in the UI (lower first); ties
// keep registration order.
- Order int `json:"order,omitempty" yaml:"order"`
+ Order int `json:"order,omitempty" yaml:"order"`
// InstanceSchema points at the JSON Schema of one instance's
// configuration (one data source, one IM channel).
InstanceSchema string `json:"instanceSchema,omitempty" yaml:"instanceSchema"`
// Path is a file or directory inside the package: the skill directory
// (skills) or the vendor definition JSON (modelVendors).
- Path string `json:"path,omitempty" yaml:"path"`
+ Path string `json:"path,omitempty" yaml:"path"`
// MCP describes a remote MCP server (mcpServers).
- MCP *MCPServer `json:"mcp,omitempty" yaml:"mcp"`
+ MCP *MCPServer `json:"mcp,omitempty" yaml:"mcp"`
// Extra carries point-specific metadata the generic fields do not cover.
- Extra map[string]any `json:"extra,omitempty" yaml:"extra"`
+ Extra map[string]any `json:"extra,omitempty" yaml:"extra"`
}
// MCPServer is a remote MCP server a plugin contributes.
type MCPServer struct {
- URL string `json:"url" yaml:"url"`
+ URL string `json:"url" yaml:"url"`
// Transport is "sse" or "http-streamable" (the default).
Transport string `json:"transport,omitempty" yaml:"transport"`
// Headers are sent on every request. A value may reference the
// workspace's plugin configuration as ${config.
} (how a plugin asks
// each workspace for its own API key) or the platform's as
// ${system.}. The URL is fixed: configuration cannot redirect it.
- Headers map[string]string `json:"headers,omitempty" yaml:"headers"`
+ Headers map[string]string `json:"headers,omitempty" yaml:"headers"`
}
// MCP transports a plugin may declare.
@@ -238,6 +238,7 @@ func (m *Manifest) Validate() error {
}
}
m.validateRuntime(add)
+ m.validatePermissions(add)
m.validateContributions(add)
return errors.Join(errs...)
}
@@ -248,8 +249,11 @@ func (m *Manifest) validateRuntime(add func(string, ...any)) {
if !m.Builtin {
add("runtime.type %q is reserved for builtin plugins", RuntimeBuiltin)
}
- case RuntimeDeclarative, RuntimeRemote, RuntimeKubernetes:
+ case RuntimeDeclarative:
+ case RuntimeRemote, RuntimeKubernetes:
+ m.validateAPIVersion(add)
case RuntimeHost:
+ m.validateAPIVersion(add)
switch m.Runtime.Kind {
case "binary", "python", "node":
default:
@@ -263,6 +267,42 @@ func (m *Manifest) validateRuntime(add func(string, ...any)) {
}
}
+// ExtensionAPIVersion is the extension protocol this build speaks
+// (pluginsdk/pluginapi.APIVersion).
+const ExtensionAPIVersion = "weknora.plugin/v1"
+
+func (m *Manifest) validateAPIVersion(add func(string, ...any)) {
+ if m.Builtin {
+ return
+ }
+ if m.APIVersion != ExtensionAPIVersion {
+ add("apiVersion must be %q for code plugins, got %q", ExtensionAPIVersion, m.APIVersion)
+ }
+}
+
+// egressPattern is a host name, optionally with a leading "*." wildcard.
+var egressPattern = regexp.MustCompile(`^(\*\.)?([a-z0-9]([a-z0-9-]*[a-z0-9])?)(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$`)
+
+// EgressAnyHost in permissions.egress grants any public host (a feed reader
+// follows links anywhere). Private addresses stay unreachable either way.
+const EgressAnyHost = "*"
+
+func (m *Manifest) validatePermissions(add func(string, ...any)) {
+ for _, e := range m.Permissions.Egress {
+ if e != EgressAnyHost && !egressPattern.MatchString(strings.ToLower(e)) {
+ add(
+ "permissions.egress %q must be a host name (api.example.com), a wildcard (*.example.com) or *",
+ e,
+ )
+ }
+ }
+}
+
+// storedTypeIDLimit is how long a qualified contribution ID may be where
+// instances store it in a varchar(50) column (data_sources.type,
+// web_search_providers.provider, knowledges.channel).
+var storedTypeIDLimit = map[Point]int{PointConnectors: 50, PointWebSearch: 50}
+
func (m *Manifest) validateContributions(add func(string, ...any)) {
if len(m.Contributes) == 0 {
add("contributes must declare at least one contribution")
@@ -297,6 +337,10 @@ func (m *Manifest) validateContributions(add func(string, ...any)) {
if c.Name.IsZero() {
add("%s.name is required", where)
}
+ if limit, ok := storedTypeIDLimit[point]; ok && !m.Builtin && len(QualifiedID(m.ID, c.ID)) > limit {
+ add("%s: %q is longer than the %d characters instances store their type in",
+ where, QualifiedID(m.ID, c.ID), limit)
+ }
if len(c.Aliases) > 0 && !m.Builtin {
add("%s.aliases may only be declared by builtin plugins", where)
}
diff --git a/internal/plugin/manifest/manifest_test.go b/internal/plugin/manifest/manifest_test.go
index a0e6d536..8a1a0bf5 100644
--- a/internal/plugin/manifest/manifest_test.go
+++ b/internal/plugin/manifest/manifest_test.go
@@ -76,7 +76,8 @@ func TestValidateReportsEveryProblem(t *testing.T) {
Version: "1.0",
Runtime: Runtime{Type: RuntimeHost},
Contributes: Contributions{
- "widgets": {{ID: "x", Name: Text("X", nil)}},
+ "widgets": {{ID: "x", Name: Text("X", nil)}},
+ PointIMChannels: {{ID: "im", Name: Text("IM", nil)}},
PointConnectors: {
{ID: "dup", Name: Text("A", nil), Aliases: []string{"old"}},
{ID: "dup"},
@@ -94,8 +95,9 @@ func TestValidateReportsEveryProblem(t *testing.T) {
"name is required",
"runtime.kind",
"runtime.entry",
+ "apiVersion must be",
"contributes.widgets is not a known extension point",
- "is not open to third-party plugins yet",
+ "contributes.imChannels is not open to third-party plugins yet",
`"dup" is declared twice`,
"contributes.connectors[1].name is required",
"aliases may only be declared by builtin plugins",
@@ -184,3 +186,35 @@ func TestContributionOmitsEmptyDescription(t *testing.T) {
t.Fatalf("empty description should be omitted: %s", data)
}
}
+
+func TestEgressPatterns(t *testing.T) {
+ base := func(egress ...string) *Manifest {
+ return &Manifest{
+ SchemaVersion: SchemaVersion, ID: "acme.x", Version: "1.0.0", APIVersion: ExtensionAPIVersion,
+ Name: Text("X", nil), Publisher: Publisher{ID: "acme"},
+ Runtime: Runtime{Type: RuntimeHost, Kind: "binary", Entry: "bin/x"},
+ Permissions: Permissions{Egress: egress},
+ Contributes: Contributions{PointWebSearch: {{ID: "x", Name: Text("X", nil)}}},
+ }
+ }
+ if err := base("api.example.com", "*.atlassian.net", "*").Validate(); err != nil {
+ t.Fatalf("valid egress rejected: %v", err)
+ }
+ for _, bad := range []string{"**", "http://x.com", "x.com:443", "*.*.com", "10.0.0.0/8"} {
+ if err := base(bad).Validate(); err == nil || !strings.Contains(err.Error(), "permissions.egress") {
+ t.Errorf("egress %q: got %v", bad, err)
+ }
+ }
+}
+
+func TestStoredTypeIDLength(t *testing.T) {
+ m := &Manifest{
+ SchemaVersion: SchemaVersion, ID: "acme.a-rather-long-plugin-name", Version: "1.0.0",
+ APIVersion: ExtensionAPIVersion, Name: Text("X", nil), Publisher: Publisher{ID: "acme"},
+ Runtime: Runtime{Type: RuntimeHost, Kind: "binary", Entry: "bin/x"},
+ Contributes: Contributions{PointConnectors: {{ID: "and-an-even-longer-connector-id", Name: Text("X", nil)}}},
+ }
+ if err := m.Validate(); err == nil || !strings.Contains(err.Error(), "longer than the 50 characters") {
+ t.Fatalf("want a length error, got %v", err)
+ }
+}
diff --git a/internal/plugin/manifest/point.go b/internal/plugin/manifest/point.go
index 92ddc30c..f3c90ccb 100644
--- a/internal/plugin/manifest/point.go
+++ b/internal/plugin/manifest/point.go
@@ -36,9 +36,9 @@ type PointInfo struct {
// infrastructure owned by WeKnora itself, not extension points.
var points = []PointInfo{
{Point: PointModelVendors, ThirdParty: true, Declarative: true},
- {Point: PointConnectors},
+ {Point: PointConnectors, ThirdParty: true},
{Point: PointIMChannels},
- {Point: PointWebSearch},
+ {Point: PointWebSearch, ThirdParty: true},
{Point: PointTools},
{Point: PointParsers},
{Point: PointSkills, ThirdParty: true, Declarative: true},
diff --git a/internal/plugin/pkg/pkg_test.go b/internal/plugin/pkg/pkg_test.go
index 9c11bb05..c40c2cb3 100644
--- a/internal/plugin/pkg/pkg_test.go
+++ b/internal/plugin/pkg/pkg_test.go
@@ -117,7 +117,7 @@ func TestOpenRejectsBadPackages(t *testing.T) {
"config/tenant.yaml": tenantSchema,
"skills/triage/SKILL.md": "x",
},
- want: "not open to third-party plugins",
+ want: "contributes.connectors needs code",
},
}
for name, c := range cases {
diff --git a/internal/plugin/reconcile/nodes.go b/internal/plugin/reconcile/nodes.go
index ec904d33..28486dce 100644
--- a/internal/plugin/reconcile/nodes.go
+++ b/internal/plugin/reconcile/nodes.go
@@ -106,23 +106,29 @@ func (r *Reconciler) NodeStatuses(ctx context.Context, pluginID string) ([]NodeS
return out, nil
}
-// Driver returns the driver for declarative plugins. They have no code to
-// run, so an instance is a node that loaded the plugin.
-func (r *Reconciler) Driver() driver.Driver { return declarativeDriver{r} }
+// Driver returns the status driver for plugins the reconciler loads on every
+// node (declarative and host): an instance is a node that loaded the plugin,
+// with the runtime health the node reported.
+func (r *Reconciler) Driver(rt manifest.RuntimeType) driver.Driver { return nodeDriver{r: r, rt: rt} }
-type declarativeDriver struct{ r *Reconciler }
-
-func (declarativeDriver) Type() manifest.RuntimeType { return manifest.RuntimeDeclarative }
-
-// Ensure and Remove are the reconciler's job for declarative plugins.
-func (declarativeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
-func (declarativeDriver) Remove(context.Context, string, string) error { return nil }
-
-func (declarativeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
- return driver.Endpoint{}, fmt.Errorf("declarative plugin %s has no endpoint", pluginID)
+type nodeDriver struct {
+ r *Reconciler
+ rt manifest.RuntimeType
}
-func (d declarativeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) {
+func (d nodeDriver) Type() manifest.RuntimeType { return d.rt }
+
+// Ensure and Remove are the reconciler's job.
+func (nodeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
+func (nodeDriver) Remove(context.Context, string, string) error { return nil }
+
+// Resolve is not how these plugins are reached: declarative ones have no
+// endpoint and host ones are reached through the node's host manager.
+func (d nodeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
+ return driver.Endpoint{}, fmt.Errorf("%s plugin %s has no shared endpoint", d.rt, pluginID)
+}
+
+func (d nodeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) {
nodes, err := d.r.NodeStatuses(ctx, pluginID)
if err != nil {
return nil, err
@@ -130,7 +136,11 @@ func (d declarativeDriver) Status(ctx context.Context, pluginID string) ([]drive
out := make([]driver.InstanceStatus, 0, len(nodes))
for _, n := range nodes {
state := driver.StateReady
- if n.State != StateReady {
+ switch n.State {
+ case StateReady:
+ case StateDegraded:
+ state = driver.StateDegraded
+ default:
state = driver.StateStopped
}
out = append(out, driver.InstanceStatus{
diff --git a/internal/plugin/reconcile/nodes_test.go b/internal/plugin/reconcile/nodes_test.go
index 34d718a3..082edae3 100644
--- a/internal/plugin/reconcile/nodes_test.go
+++ b/internal/plugin/reconcile/nodes_test.go
@@ -8,6 +8,7 @@ import (
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
+ "github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/plugintest"
"github.com/Tencent/WeKnora/internal/plugin/registry"
"github.com/Tencent/WeKnora/internal/types"
@@ -45,7 +46,7 @@ func TestNodesConvergeThroughRedis(t *testing.T) {
nodes, err := a.NodeStatuses(ctx, "acme.kit")
return err == nil && len(nodes) == 2
})
- instances, err := b.Driver().Status(ctx, "acme.kit")
+ instances, err := b.Driver(manifest.RuntimeDeclarative).Status(ctx, "acme.kit")
if err != nil || len(instances) != 2 || instances[0].State != "ready" {
t.Fatalf("instances = %+v, %v", instances, err)
}
diff --git a/internal/plugin/reconcile/reconcile.go b/internal/plugin/reconcile/reconcile.go
index 3311ef25..2b122ccb 100644
--- a/internal/plugin/reconcile/reconcile.go
+++ b/internal/plugin/reconcile/reconcile.go
@@ -49,6 +49,14 @@ type Activator interface {
Deactivate(ctx context.Context, pluginID string) error
}
+// InPlaceActivator swaps an upgraded plugin itself inside Activate (the host
+// starts the new process before stopping the old one), so it is not
+// deactivated first.
+type InPlaceActivator interface {
+ Activator
+ ActivatesInPlace()
+}
+
// Status is how a plugin fares on this node.
type Status struct {
Version string `json:"version"`
@@ -61,6 +69,9 @@ type Status struct {
const (
StateReady = "ready"
StateFailed = "failed"
+ // StateDegraded: loaded, but its process crashed or fails health checks
+ // and is being restarted.
+ StateDegraded = "degraded"
)
// DefaultInterval is how often a node reconciles without being told to.
@@ -208,7 +219,8 @@ func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) erro
l := &Loaded{Manifest: p.Manifest, Package: p, Dir: dir}
var errs []error
for _, a := range r.activators {
- if _, had := r.loaded[row.ID]; had {
+ _, inPlace := a.(InPlaceActivator)
+ if _, had := r.loaded[row.ID]; had && !inPlace {
if err := a.Deactivate(ctx, row.ID); err != nil {
errs = append(errs, fmt.Errorf("%s: deactivate previous version: %w", a.Name(), err))
}
@@ -228,7 +240,9 @@ func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) erro
}
func (r *Reconciler) unload(ctx context.Context, id string) {
- for _, a := range r.activators {
+ // Reverse order: routes go before the processes they route to.
+ for i := len(r.activators) - 1; i >= 0; i-- {
+ a := r.activators[i]
if err := a.Deactivate(ctx, id); err != nil {
logger.Warnf(ctx, "[plugin] %s: deactivate %s: %v", a.Name(), id, err)
}
@@ -286,6 +300,28 @@ func (r *Reconciler) setStatus(id string, s Status) {
r.statusMu.Unlock()
}
+// ReportRuntime records a loaded plugin's runtime health (a host process that
+// crashed and is restarting, or recovered) and publishes it at once.
+func (r *Reconciler) ReportRuntime(pluginID string, healthy bool, err error) {
+ r.statusMu.Lock()
+ s, ok := r.status[pluginID]
+ if !ok || s.State == StateFailed {
+ r.statusMu.Unlock()
+ return
+ }
+ s.State, s.Error = StateReady, ""
+ if !healthy {
+ s.State = StateDegraded
+ if err != nil {
+ s.Error = err.Error()
+ }
+ }
+ s.UpdatedAt = time.Now()
+ r.status[pluginID] = s
+ r.statusMu.Unlock()
+ r.publishStatuses(context.Background())
+}
+
// Status reports how one plugin fares on this node.
func (r *Reconciler) Status(pluginID string) (Status, bool) {
r.statusMu.RLock()
diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go
index cdbb92a7..abc5e745 100644
--- a/internal/types/interfaces/datasource.go
+++ b/internal/types/interfaces/datasource.go
@@ -41,7 +41,11 @@ type DataSourceService interface {
// ValidateCredentials tests connectivity using raw credentials without persisting anything.
// This is used by the frontend "Test Connection" button before creating a data source.
- ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error
+ // ValidateCredentials tests unsaved credentials (and settings, which
+ // some connectors need to reach the source) against the connector.
+ ValidateCredentials(
+ ctx context.Context, connectorType string, credentials, settings map[string]interface{},
+ ) error
// ListAvailableResources lists resources available for sync in the external system.
// parentID enables lazy loading: "" lists the top level, a resource ExternalID lists its children.
diff --git a/internal/types/web_search_provider.go b/internal/types/web_search_provider.go
index b71c115a..ddf35912 100644
--- a/internal/types/web_search_provider.go
+++ b/internal/types/web_search_provider.go
@@ -134,6 +134,13 @@ type WebSearchProviderTypeInfo struct {
ID string `json:"id"`
// Human-readable name
Name string `json:"name"`
+ // Names localizes Name for plugin provider types, keyed by locale.
+ Names map[string]string `json:"names,omitempty"`
+ // Icon is an image (data: URI) for plugin provider types; builtin types
+ // use the frontend's bundled logos.
+ Icon string `json:"icon,omitempty"`
+ // PluginID names the installed plugin providing the type.
+ PluginID string `json:"plugin_id,omitempty"`
// Whether the provider requires an API key
RequiresAPIKey bool `json:"requires_api_key"`
// Whether the provider accepts an optional API key (keyless by default, but a
diff --git a/pluginsdk/README.md b/pluginsdk/README.md
new file mode 100644
index 00000000..6cd65e16
--- /dev/null
+++ b/pluginsdk/README.md
@@ -0,0 +1,120 @@
+# WeKnora plugin SDK (Go)
+
+Build code plugins for WeKnora: web search providers and data source
+connectors that run as their own process. The module has no dependencies.
+
+| Package | What it is |
+| --- | --- |
+| `pluginapi` | The extension protocol v1: envelope, errors, stream events, messages. `openapi.yaml` describes it for other languages. |
+| `pluginsdk` | Write a plugin: register contributions, then `Serve`. |
+| `client` | Call a plugin, as WeKnora does. |
+| `conformance`, `cmd/weknora-plugin-conformance` | Check that a plugin speaks the protocol. |
+
+## A connector in 30 lines
+
+```go
+package main
+
+import (
+ "context"
+ "log"
+
+ "github.com/Tencent/WeKnora/pluginsdk"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+type notes struct{}
+
+func (notes) Validate(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
+ if cfg.Credentials["token"] == nil {
+ return pluginapi.InvalidConfig("token is required", map[string]string{"credentials.token": "required"})
+ }
+ return nil
+}
+
+func (notes) ListResources(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig, parent string) ([]pluginapi.Resource, error) {
+ return []pluginapi.Resource{{ExternalID: "inbox", Name: "Inbox"}}, nil
+}
+
+func (notes) Fetch(ctx context.Context, call *pluginsdk.Call, cfg pluginsdk.ConnectorConfig, in pluginapi.FetchInput, s *pluginsdk.Stream) (*pluginapi.Cursor, error) {
+ if err := s.Item(pluginapi.FetchedItem{ExternalID: "n1", Title: "First note", Content: []byte("# Hello")}); err != nil {
+ return nil, err
+ }
+ return &pluginapi.Cursor{State: map[string]any{"after": "n1"}}, nil
+}
+
+func main() {
+ p := pluginsdk.New(pluginsdk.Info{ID: "acme.notes", Version: "1.0.0"})
+ p.Connector("notes", notes{})
+ log.Fatal(p.Serve())
+}
+```
+
+What the SDK does for you:
+
+- **Handshake.** Started by WeKnora's plugin host, the plugin listens on the
+ socket it is given and accepts only the host's token.
+- **Remote mode.** Run on its own, it listens on `WEKNORA_PLUGIN_ADDR` and
+ verifies signed requests with `WEKNORA_PLUGIN_SECRET`.
+- **Per call.** Each call's deadline reaches your context. Returned
+ `pluginapi` errors keep their code; anything else, and any panic, becomes
+ `internal`.
+
+Every call carries the tenant and the configuration, secrets already
+decrypted, in three separate scopes:
+
+- `call.Config.System`: set by the system administrator.
+- `call.Config.Tenant`: set per workspace.
+- the instance: `cfg`, or `call.DecodeInstance`.
+
+Keep plugins stateless and never cache a tenant's credentials across calls.
+
+## Packaging
+
+A package is a zip (`.wkp`) with `plugin.yaml` at its root:
+
+```yaml
+schemaVersion: 1
+id: acme.notes
+version: 1.0.0
+apiVersion: weknora.plugin/v1
+name: { en-US: ACME Notes }
+publisher: { id: acme }
+runtime: { type: host, kind: binary, entry: "bin/{os}-{arch}/notes" }
+permissions:
+ egress: [api.acme.example] # hosts the plugin may reach; "*" = any public host
+contributes:
+ connectors:
+ - id: notes
+ name: { en-US: ACME Notes }
+ instanceSchema: schemas/notes.yaml # fields with x-group: settings are settings, the rest credentials
+```
+
+A few rules the manifest and host enforce:
+
+- **Binaries.** Build one per platform into `bin/-/`; WeKnora
+ runs the build for its own OS and architecture.
+- **Environment.** The process gets no environment from WeKnora beyond the
+ `WEKNORA_PLUGIN_*` variables.
+- **Outbound traffic.** It goes through the host's egress proxy, which
+ forwards only to the hosts in `permissions.egress` and never to private
+ addresses.
+- **IDs.** Contribution IDs are qualified with the plugin ID
+ (`acme.notes/notes`); for connectors and web search that must stay within
+ 50 characters.
+
+See `examples/plugins/rss` for a complete plugin and its `package.sh`.
+
+## Testing
+
+```bash
+go run github.com/Tencent/WeKnora/pluginsdk/cmd/weknora-plugin-conformance -url http://localhost:8080 -secret $SECRET
+```
+
+In Go tests, run `conformance.Run` against `httptest.NewServer(p.Handler())`.
+
+## Installing
+
+In WeKnora: **System administration → Plugin management → Install plugin**.
+Upload the `.wkp` or give its URL, review what it adds and reaches, and
+install. Each workspace then enables it under **Settings → Plugins**.
diff --git a/pluginsdk/client/client.go b/pluginsdk/client/client.go
new file mode 100644
index 00000000..e0c2de4b
--- /dev/null
+++ b/pluginsdk/client/client.go
@@ -0,0 +1,259 @@
+// Package client calls a WeKnora code plugin over the extension protocol. It
+// is what WeKnora itself uses, and what the conformance suite tests plugins
+// with.
+package client
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// Auth adds credentials to a request.
+type Auth interface {
+ Apply(req *http.Request, body []byte) error
+}
+
+// Bearer authenticates as the plugin host.
+type Bearer string
+
+// Apply implements Auth.
+func (b Bearer) Apply(req *http.Request, _ []byte) error {
+ req.Header.Set("Authorization", "Bearer "+string(b))
+ return nil
+}
+
+// Signed authenticates to a remote plugin with its shared secret.
+type Signed []byte
+
+// Apply implements Auth.
+func (s Signed) Apply(req *http.Request, body []byte) error {
+ ts := time.Now().Unix()
+ req.Header.Set(pluginapi.TimestampHeader, strconv.FormatInt(ts, 10))
+ req.Header.Set(pluginapi.SignatureHeader, pluginapi.Sign(s, ts, body))
+ return nil
+}
+
+// TransportError is a failure without a protocol answer: the plugin was
+// unreachable, answered something that is not the protocol, or broke off a
+// stream. It unwraps to a *pluginapi.Error (unavailable or internal) so
+// callers can treat every failure alike.
+type TransportError struct {
+ Err *pluginapi.Error
+}
+
+func (e *TransportError) Error() string { return e.Err.Error() }
+func (e *TransportError) Unwrap() error { return e.Err }
+
+func transport(code pluginapi.ErrorCode, format string, args ...any) error {
+ return &TransportError{Err: &pluginapi.Error{
+ Code: code, Message: fmt.Sprintf(format, args...), Retryable: code == pluginapi.CodeUnavailable,
+ }}
+}
+
+// Client calls one plugin process or service.
+type Client struct {
+ base string
+ http *http.Client
+ auth Auth
+}
+
+// ForHandshake reaches a host plugin at the address its handshake names.
+// The returned client has no overall timeout: calls are bounded by their
+// context.
+func ForHandshake(h pluginapi.Handshake, auth Auth) *Client {
+ if h.Network == "unix" {
+ tr := &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, "unix", h.Address)
+ },
+ MaxIdleConnsPerHost: 16,
+ IdleConnTimeout: 90 * time.Second,
+ }
+ return &Client{base: "http://plugin", http: &http.Client{Transport: tr}, auth: auth}
+ }
+ return New("http://"+h.Address, nil, auth)
+}
+
+// New reaches a plugin at an HTTP(S) base URL. A nil httpClient uses a
+// client without an overall timeout.
+func New(baseURL string, httpClient *http.Client, auth Auth) *Client {
+ if httpClient == nil {
+ httpClient = &http.Client{}
+ }
+ return &Client{base: strings.TrimSuffix(baseURL, "/"), http: httpClient, auth: auth}
+}
+
+// Close releases idle connections.
+func (c *Client) Close() { c.http.CloseIdleConnections() }
+
+func (c *Client) do(ctx context.Context, method, path string, body []byte, requestID string) (*http.Response, error) {
+ var rd io.Reader
+ if body != nil {
+ rd = bytes.NewReader(body)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, c.base+path, rd)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set(pluginapi.ProtocolHeader, pluginapi.ProtocolVersion)
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ if requestID != "" {
+ req.Header.Set(pluginapi.RequestIDHeader, requestID)
+ }
+ if c.auth != nil {
+ if err := c.auth.Apply(req, body); err != nil {
+ return nil, err
+ }
+ }
+ resp, err := c.http.Do(req)
+ if err != nil {
+ return nil, transport(pluginapi.CodeUnavailable, "%v", err)
+ }
+ return resp, nil
+}
+
+// decodeError turns a non-2xx answer into a protocol error.
+func decodeError(resp *http.Response) error {
+ b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
+ var body pluginapi.ErrorBody
+ if json.Unmarshal(b, &body) == nil && body.Error.Code != "" {
+ return &body.Error
+ }
+ code := pluginapi.CodeInternal
+ if resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusBadGateway {
+ code = pluginapi.CodeUnavailable
+ }
+ return transport(code, "plugin answered HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
+}
+
+// Get reads a GET endpoint (manifest, health) into out.
+func (c *Client) Get(ctx context.Context, path string, out any) error {
+ resp, err := c.do(ctx, http.MethodGet, path, nil, "")
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode/100 != 2 {
+ return decodeError(resp)
+ }
+ return json.NewDecoder(resp.Body).Decode(out)
+}
+
+// Manifest reads GET /v1/manifest.
+func (c *Client) Manifest(ctx context.Context) (*pluginapi.Manifest, error) {
+ var m pluginapi.Manifest
+ return &m, c.Get(ctx, "/v1/manifest", &m)
+}
+
+// Health reads GET /v1/health.
+func (c *Client) Health(ctx context.Context) error {
+ var h pluginapi.Health
+ if err := c.Get(ctx, "/v1/health", &h); err != nil {
+ return err
+ }
+ if h.Status != "ok" {
+ return &pluginapi.Error{Code: pluginapi.CodeUnavailable, Message: "plugin reports " + h.Status, Retryable: true}
+ }
+ return nil
+}
+
+func envelopeBody(env pluginapi.Envelope, input any) ([]byte, error) {
+ if input != nil {
+ raw, err := json.Marshal(input)
+ if err != nil {
+ return nil, err
+ }
+ env.Input = raw
+ }
+ return json.Marshal(env)
+}
+
+// Call posts an envelope and decodes the output into out (nil discards it).
+func (c *Client) Call(ctx context.Context, path string, env pluginapi.Envelope, input, out any) error {
+ body, err := envelopeBody(env, input)
+ if err != nil {
+ return err
+ }
+ resp, err := c.do(ctx, http.MethodPost, path, body, env.Context.RequestID)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode/100 != 2 {
+ return decodeError(resp)
+ }
+ var wrapped pluginapi.Output
+ if err := json.NewDecoder(resp.Body).Decode(&wrapped); err != nil {
+ return transport(pluginapi.CodeInternal, "decode plugin output: %v", err)
+ }
+ if out == nil || len(wrapped.Output) == 0 {
+ return nil
+ }
+ return json.Unmarshal(wrapped.Output, out)
+}
+
+// MaxEventBytes bounds one NDJSON line (one fetched item).
+const MaxEventBytes = 64 << 20
+
+// Stream posts an envelope and hands each event to fn until the stream ends.
+// It returns the "end" event's data, the "error" event as an error, or an
+// error when the stream stops without either.
+func (c *Client) Stream(
+ ctx context.Context, path string, env pluginapi.Envelope, input any,
+ fn func(pluginapi.Event) error,
+) (json.RawMessage, error) {
+ body, err := envelopeBody(env, input)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := c.do(ctx, http.MethodPost, path, body, env.Context.RequestID)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode/100 != 2 {
+ return nil, decodeError(resp)
+ }
+ sc := bufio.NewScanner(resp.Body)
+ sc.Buffer(make([]byte, 0, 64<<10), MaxEventBytes)
+ for sc.Scan() {
+ line := bytes.TrimSpace(sc.Bytes())
+ if len(line) == 0 {
+ continue
+ }
+ var ev pluginapi.Event
+ if err := json.Unmarshal(line, &ev); err != nil {
+ return nil, transport(pluginapi.CodeInternal, "decode stream event: %v", err)
+ }
+ switch ev.Type {
+ case pluginapi.EventEnd:
+ return ev.Data, nil
+ case pluginapi.EventError:
+ if ev.Error == nil {
+ return nil, &pluginapi.Error{Code: pluginapi.CodeInternal, Message: "plugin stream failed"}
+ }
+ return nil, ev.Error
+ }
+ if err := fn(ev); err != nil {
+ return nil, err
+ }
+ }
+ if err := sc.Err(); err != nil {
+ return nil, transport(pluginapi.CodeUnavailable, "stream broken: %v", err)
+ }
+ return nil, transport(pluginapi.CodeUnavailable, "stream ended without an end event")
+}
diff --git a/pluginsdk/cmd/weknora-plugin-conformance/main.go b/pluginsdk/cmd/weknora-plugin-conformance/main.go
new file mode 100644
index 00000000..8c1cd2af
--- /dev/null
+++ b/pluginsdk/cmd/weknora-plugin-conformance/main.go
@@ -0,0 +1,83 @@
+// Command weknora-plugin-conformance checks a running plugin against the
+// extension protocol.
+//
+// weknora-plugin-conformance -url https://plugin.example.com -secret $SECRET
+// weknora-plugin-conformance -socket /tmp/p.sock -token $TOKEN
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/conformance"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+func main() {
+ url := flag.String("url", "", "base URL of a remote plugin")
+ socket := flag.String("socket", "", "unix socket of a plugin started by hand in host mode")
+ token := flag.String("token", "", "host token (with -socket)")
+ secret := flag.String("secret", "", "shared secret (with -url)")
+ asJSON := flag.Bool("json", false, "print the report as JSON")
+ timeout := flag.Duration("timeout", 30*time.Second, "per-check timeout")
+ flag.Parse()
+
+ var c, anon *client.Client
+ var raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
+ switch {
+ case *socket != "":
+ hs := pluginapi.Handshake{Protocol: pluginapi.ProtocolVersion, Network: "unix", Address: *socket}
+ c = client.ForHandshake(hs, client.Bearer(*token))
+ anon = client.ForHandshake(hs, nil)
+ case *url != "":
+ var auth client.Auth
+ if *secret != "" {
+ auth = client.Signed(*secret)
+ anon = client.New(*url, nil, nil)
+ }
+ c = client.New(*url, nil, auth)
+ raw = func(ctx context.Context, path string, body []byte) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, *url+path, bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ if auth != nil {
+ _ = auth.Apply(req, body)
+ }
+ return http.DefaultClient.Do(req)
+ }
+ default:
+ fmt.Fprintln(os.Stderr, "give -url or -socket")
+ os.Exit(2)
+ }
+
+ rep := conformance.Run(context.Background(), conformance.Target{
+ Client: c, Unauthenticated: anon, Raw: raw, Timeout: *timeout,
+ })
+ if *asJSON {
+ _ = json.NewEncoder(os.Stdout).Encode(rep)
+ } else {
+ fmt.Printf("plugin %s\n", rep.Plugin)
+ for _, r := range rep.Results {
+ mark := "PASS"
+ if !r.Passed {
+ mark = "FAIL"
+ }
+ fmt.Printf(" %s %s", mark, r.Name)
+ if r.Detail != "" {
+ fmt.Printf(" — %s", r.Detail)
+ }
+ fmt.Println()
+ }
+ }
+ if !rep.Passed() {
+ os.Exit(1)
+ }
+}
diff --git a/pluginsdk/conformance/conformance.go b/pluginsdk/conformance/conformance.go
new file mode 100644
index 00000000..a24d8729
--- /dev/null
+++ b/pluginsdk/conformance/conformance.go
@@ -0,0 +1,233 @@
+// Package conformance checks that a plugin speaks the extension protocol:
+// it answers the discovery endpoints, rejects what it must reject, and
+// answers every endpoint it claims with either a well-formed output or a
+// well-formed protocol error. It does not judge the answers themselves; a
+// plugin's own tests do that.
+package conformance
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "regexp"
+ "time"
+
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// Result is one check.
+type Result struct {
+ Name string `json:"name"`
+ Passed bool `json:"passed"`
+ Detail string `json:"detail,omitempty"`
+}
+
+// Report is every check against one plugin.
+type Report struct {
+ Plugin string `json:"plugin"`
+ Results []Result `json:"results"`
+}
+
+// Passed reports whether every check passed.
+func (r Report) Passed() bool {
+ for _, res := range r.Results {
+ if !res.Passed {
+ return false
+ }
+ }
+ return len(r.Results) > 0
+}
+
+// Target is the plugin under test.
+type Target struct {
+ // Client reaches the plugin with valid credentials.
+ Client *client.Client
+ // Unauthenticated reaches it without credentials; nil skips the check
+ // (a plugin served without authentication in a test harness).
+ Unauthenticated *client.Client
+ // Raw posts arbitrary bytes, for malformed-request checks; nil skips.
+ Raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
+ // Timeout bounds each call (default 30s).
+ Timeout time.Duration
+}
+
+var semver = regexp.MustCompile(`^\d+\.\d+\.\d+([-+].*)?$`)
+
+// Run checks a plugin.
+func Run(ctx context.Context, t Target) Report {
+ if t.Timeout <= 0 {
+ t.Timeout = 30 * time.Second
+ }
+ var rep Report
+ check := func(name string, fn func(ctx context.Context) error) {
+ cctx, cancel := context.WithTimeout(ctx, t.Timeout)
+ defer cancel()
+ res := Result{Name: name, Passed: true}
+ if err := fn(cctx); err != nil {
+ res.Passed, res.Detail = false, err.Error()
+ }
+ rep.Results = append(rep.Results, res)
+ }
+
+ var m *pluginapi.Manifest
+ check("manifest", func(ctx context.Context) error {
+ var err error
+ if m, err = t.Client.Manifest(ctx); err != nil {
+ return err
+ }
+ rep.Plugin = m.ID + "@" + m.Version
+ switch {
+ case m.ID == "":
+ return fmt.Errorf("manifest has no id")
+ case !semver.MatchString(m.Version):
+ return fmt.Errorf("version %q is not semantic", m.Version)
+ case m.APIVersion != pluginapi.APIVersion:
+ return fmt.Errorf("apiVersion %q, want %q", m.APIVersion, pluginapi.APIVersion)
+ case len(m.Contributes) == 0:
+ return fmt.Errorf("manifest lists no contributions")
+ }
+ return nil
+ })
+ check("health", func(ctx context.Context) error { return t.Client.Health(ctx) })
+ if t.Unauthenticated != nil {
+ check("rejects unauthenticated calls", func(ctx context.Context) error {
+ return wantCode(t.Unauthenticated.Health(ctx), pluginapi.CodeUnauthorized)
+ })
+ }
+ check("unknown endpoint answers not_found", func(ctx context.Context) error {
+ return wantCode(
+ t.Client.Call(ctx, "/v1/no-such-endpoint", pluginapi.Envelope{}, nil, nil),
+ pluginapi.CodeNotFound,
+ )
+ })
+ check("unknown contribution answers not_found", func(ctx context.Context) error {
+ return wantCode(t.Client.Call(ctx, pluginapi.SearchPath("no-such-provider"), pluginapi.Envelope{},
+ pluginapi.SearchInput{Query: "x"}, nil), pluginapi.CodeNotFound)
+ })
+ if t.Raw != nil {
+ check("malformed envelope answers bad_request", func(ctx context.Context) error {
+ resp, err := t.Raw(ctx, "/v1/config/validate", []byte("{not json"))
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ return wantErrorBody(resp, pluginapi.CodeBadRequest)
+ })
+ }
+ check("config/validate answers", func(ctx context.Context) error {
+ return protocolAnswer(t.Client.Call(ctx, "/v1/config/validate", envelope(), nil, nil))
+ })
+ if m == nil {
+ return rep
+ }
+ for _, id := range m.Contributes["webSearch"] {
+ check("webSearch/"+id+" search answers", func(ctx context.Context) error {
+ var out pluginapi.SearchOutput
+ err := t.Client.Call(
+ ctx,
+ pluginapi.SearchPath(id),
+ envelope(),
+ pluginapi.SearchInput{Query: "weknora", MaxResults: 3},
+ &out,
+ )
+ if err == nil && out.Results == nil {
+ return fmt.Errorf("output has no results array")
+ }
+ return protocolAnswer(err)
+ })
+ }
+ for _, id := range m.Contributes["connectors"] {
+ check("connectors/"+id+" validate answers", func(ctx context.Context) error {
+ return protocolAnswer(t.Client.Call(ctx, pluginapi.ConnectorValidatePath(id), envelope(), nil, nil))
+ })
+ check("connectors/"+id+" list-resources answers", func(ctx context.Context) error {
+ var out pluginapi.ListResourcesOutput
+ err := t.Client.Call(
+ ctx,
+ pluginapi.ConnectorListResourcesPath(id),
+ envelope(),
+ pluginapi.ListResourcesInput{},
+ &out,
+ )
+ if err == nil && out.Resources == nil {
+ return fmt.Errorf("output has no resources array")
+ }
+ return protocolAnswer(err)
+ })
+ check("connectors/"+id+" resolve-ancestors answers", func(ctx context.Context) error {
+ var out pluginapi.ResolveAncestorsOutput
+ err := t.Client.Call(ctx, pluginapi.ConnectorResolveAncestorsPath(id), envelope(),
+ pluginapi.ResolveAncestorsInput{ResourceIDs: []string{}}, &out)
+ if err == nil && out.Ancestors == nil {
+ return fmt.Errorf("output has no ancestors array")
+ }
+ return protocolAnswer(err)
+ })
+ check("connectors/"+id+" fetch stream terminates", func(ctx context.Context) error {
+ _, err := t.Client.Stream(ctx, pluginapi.ConnectorFetchPath(id), envelope(),
+ pluginapi.FetchInput{Mode: pluginapi.FetchFull}, func(ev pluginapi.Event) error {
+ switch ev.Type {
+ case pluginapi.EventItem, pluginapi.EventCheckpoint, pluginapi.EventProgress, pluginapi.EventLog:
+ return nil
+ }
+ return fmt.Errorf("unknown event type %q", ev.Type)
+ })
+ return protocolAnswer(err)
+ })
+ }
+ return rep
+}
+
+// envelope is a call with empty configuration: plugins must answer it with
+// an output or a protocol error, never a crash or garbage.
+func envelope() pluginapi.Envelope {
+ return pluginapi.Envelope{Context: pluginapi.Context{TenantID: 1, RequestID: "conformance", Locale: "en-US"}}
+}
+
+// protocolAnswer accepts success or a protocol error the plugin chose, and
+// fails transport failures and internal errors: a plugin must answer an
+// empty configuration with invalid_config, not by crashing.
+func protocolAnswer(err error) error {
+ if err == nil {
+ return nil
+ }
+ var te *client.TransportError
+ if errors.As(err, &te) {
+ return fmt.Errorf("no protocol answer: %v", te)
+ }
+ e, ok := pluginapi.AsError(err)
+ if !ok {
+ return err
+ }
+ if e.Code == pluginapi.CodeInternal {
+ return fmt.Errorf("answered internal: %s", e.Message)
+ }
+ return nil
+}
+
+func wantCode(err error, code pluginapi.ErrorCode) error {
+ e, ok := pluginapi.AsError(err)
+ if !ok {
+ return fmt.Errorf("want a %s protocol error, got %v", code, err)
+ }
+ if e.Code != code {
+ return fmt.Errorf("want %s, got %s: %s", code, e.Code, e.Message)
+ }
+ return nil
+}
+
+func wantErrorBody(resp *http.Response, code pluginapi.ErrorCode) error {
+ b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
+ var body pluginapi.ErrorBody
+ if err := json.Unmarshal(b, &body); err != nil {
+ return fmt.Errorf("HTTP %d without a protocol error body: %s", resp.StatusCode, b)
+ }
+ if resp.StatusCode != code.HTTPStatus() || body.Error.Code != code {
+ return fmt.Errorf("want HTTP %d %s, got HTTP %d %s", code.HTTPStatus(), code, resp.StatusCode, body.Error.Code)
+ }
+ return nil
+}
diff --git a/pluginsdk/conformance/conformance_test.go b/pluginsdk/conformance/conformance_test.go
new file mode 100644
index 00000000..d4ca866a
--- /dev/null
+++ b/pluginsdk/conformance/conformance_test.go
@@ -0,0 +1,105 @@
+package conformance_test
+
+import (
+ "bytes"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/pluginsdk"
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/conformance"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+type strictConnector struct{}
+
+func (strictConnector) Validate(_ context.Context, _ *pluginsdk.Call, cfg pluginsdk.ConnectorConfig) error {
+ if cfg.Settings["url"] == nil {
+ return pluginapi.InvalidConfig("url is required", map[string]string{"settings.url": "required"})
+ }
+ return nil
+}
+
+func (s strictConnector) ListResources(
+ ctx context.Context,
+ call *pluginsdk.Call,
+ cfg pluginsdk.ConnectorConfig,
+ _ string,
+) ([]pluginapi.Resource, error) {
+ return nil, s.Validate(ctx, call, cfg)
+}
+
+func (s strictConnector) Fetch(
+ ctx context.Context,
+ call *pluginsdk.Call,
+ cfg pluginsdk.ConnectorConfig,
+ _ pluginapi.FetchInput,
+ _ *pluginsdk.Stream,
+) (*pluginapi.Cursor, error) {
+ return nil, s.Validate(ctx, call, cfg)
+}
+
+func target(t *testing.T, h http.Handler) conformance.Target {
+ srv := httptest.NewServer(h)
+ t.Cleanup(srv.Close)
+ return conformance.Target{
+ Client: client.New(srv.URL, nil, nil),
+ Raw: func(ctx context.Context, path string, body []byte) (*http.Response, error) {
+ req, _ := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL+path, bytes.NewReader(body))
+ return http.DefaultClient.Do(req)
+ },
+ }
+}
+
+func TestSDKPluginConforms(t *testing.T) {
+ p := pluginsdk.New(pluginsdk.Info{ID: "acme.strict", Version: "1.0.0"})
+ p.Connector("strict", strictConnector{})
+ p.WebSearch(
+ "none",
+ pluginsdk.WebSearchFunc(
+ func(context.Context, *pluginsdk.Call, pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
+ return nil, pluginapi.InvalidConfig("api key required", map[string]string{"api_key": "required"})
+ },
+ ),
+ )
+ rep := conformance.Run(context.Background(), target(t, p.Handler()))
+ for _, r := range rep.Results {
+ if !r.Passed {
+ t.Errorf("%s: %s", r.Name, r.Detail)
+ }
+ }
+ if !rep.Passed() || rep.Plugin != "acme.strict@1.0.0" || len(rep.Results) < 10 {
+ t.Fatalf("report = %+v", rep)
+ }
+}
+
+func TestBrokenPluginFails(t *testing.T) {
+ p := pluginsdk.New(pluginsdk.Info{ID: "acme.broken", Version: "1.0.0"})
+ p.Connector("broken", strictConnector{})
+ good := p.Handler()
+ // A fetch that returns plain text and a list that crashes.
+ broken := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/fetch"):
+ _, _ = w.Write([]byte("oops"))
+ case strings.HasSuffix(r.URL.Path, "/list-resources"):
+ panic("nil map")
+ default:
+ good.ServeHTTP(w, r)
+ }
+ })
+ rep := conformance.Run(context.Background(), target(t, broken))
+ failed := map[string]bool{}
+ for _, r := range rep.Results {
+ if !r.Passed {
+ failed[r.Name] = true
+ }
+ }
+ if rep.Passed() || !failed["connectors/broken fetch stream terminates"] ||
+ !failed["connectors/broken list-resources answers"] {
+ t.Fatalf("failed checks = %v", failed)
+ }
+}
diff --git a/pluginsdk/connector.go b/pluginsdk/connector.go
new file mode 100644
index 00000000..0013c6e6
--- /dev/null
+++ b/pluginsdk/connector.go
@@ -0,0 +1,160 @@
+package pluginsdk
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// ConnectorConfig is a data source's instance configuration.
+type ConnectorConfig struct {
+ Credentials map[string]any `json:"credentials"`
+ Settings map[string]any `json:"settings"`
+ ResourceIDs []string `json:"resourceIds"`
+}
+
+// Connector is a data source connector.
+type Connector interface {
+ // Validate checks the configuration, typically by calling the source.
+ // Return pluginapi.InvalidConfig to point at fields.
+ Validate(ctx context.Context, call *Call, cfg ConnectorConfig) error
+ // ListResources lists what can be synced under parentID ("" = top).
+ ListResources(ctx context.Context, call *Call, cfg ConnectorConfig, parentID string) ([]pluginapi.Resource, error)
+ // Fetch streams items changed since cursor (nil = everything) and
+ // returns the cursor for the next sync. Checkpoint at page boundaries
+ // so an interrupted sync resumes.
+ Fetch(
+ ctx context.Context,
+ call *Call,
+ cfg ConnectorConfig,
+ in pluginapi.FetchInput,
+ s *Stream,
+ ) (*pluginapi.Cursor, error)
+}
+
+// AncestorResolver is implemented by connectors whose resource tree loads
+// lazily; see pluginapi.ResolveAncestorsInput. Others answer with nothing.
+type AncestorResolver interface {
+ ResolveAncestors(ctx context.Context, call *Call, cfg ConnectorConfig, resourceIDs []string) ([]string, error)
+}
+
+// Connector registers connector id (contributes.connectors[].id).
+func (p *Plugin) Connector(id string, c Connector) { p.connectors[id] = c }
+
+func (p *Plugin) connector(w http.ResponseWriter, r *http.Request) (Connector, bool) {
+ c, ok := p.connectors[r.PathValue("id")]
+ if !ok {
+ writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no connector %q", r.PathValue("id")))
+ }
+ return c, ok
+}
+
+func connectorConfig(call *Call) (ConnectorConfig, error) {
+ var cfg ConnectorConfig
+ if err := call.DecodeInstance(&cfg); err != nil {
+ return cfg, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode connector config: %v", err)
+ }
+ return cfg, nil
+}
+
+func (p *Plugin) routeConnectors(mux *http.ServeMux) {
+ mux.HandleFunc("POST /v1/connectors/{id}/validate", func(w http.ResponseWriter, r *http.Request) {
+ c, ok := p.connector(w, r)
+ if !ok {
+ return
+ }
+ p.unary(func(ctx context.Context, call *Call, _ json.RawMessage) (any, error) {
+ cfg, err := connectorConfig(call)
+ if err != nil {
+ return nil, err
+ }
+ return struct{}{}, c.Validate(ctx, call, cfg)
+ })(w, r)
+ })
+ mux.HandleFunc("POST /v1/connectors/{id}/list-resources", func(w http.ResponseWriter, r *http.Request) {
+ c, ok := p.connector(w, r)
+ if !ok {
+ return
+ }
+ p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
+ in, err := decodeInput[pluginapi.ListResourcesInput](raw)
+ if err != nil {
+ return nil, err
+ }
+ cfg, err := connectorConfig(call)
+ if err != nil {
+ return nil, err
+ }
+ res, err := c.ListResources(ctx, call, cfg, in.ParentID)
+ if err != nil {
+ return nil, err
+ }
+ if res == nil {
+ res = []pluginapi.Resource{}
+ }
+ return pluginapi.ListResourcesOutput{Resources: res}, nil
+ })(w, r)
+ })
+ mux.HandleFunc("POST /v1/connectors/{id}/resolve-ancestors", func(w http.ResponseWriter, r *http.Request) {
+ c, ok := p.connector(w, r)
+ if !ok {
+ return
+ }
+ p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
+ in, err := decodeInput[pluginapi.ResolveAncestorsInput](raw)
+ if err != nil {
+ return nil, err
+ }
+ out := pluginapi.ResolveAncestorsOutput{Ancestors: []string{}}
+ resolver, ok := c.(AncestorResolver)
+ if !ok {
+ return out, nil
+ }
+ cfg, err := connectorConfig(call)
+ if err != nil {
+ return nil, err
+ }
+ ids, err := resolver.ResolveAncestors(ctx, call, cfg, in.ResourceIDs)
+ if err != nil {
+ return nil, err
+ }
+ if ids != nil {
+ out.Ancestors = ids
+ }
+ return out, nil
+ })(w, r)
+ })
+ mux.HandleFunc("POST /v1/connectors/{id}/fetch", func(w http.ResponseWriter, r *http.Request) {
+ c, ok := p.connector(w, r)
+ if !ok {
+ return
+ }
+ env, err := readEnvelope(r)
+ if err != nil {
+ writeError(w, err)
+ return
+ }
+ in, err := decodeInput[pluginapi.FetchInput](env.Input)
+ if err != nil {
+ writeError(w, err)
+ return
+ }
+ call := &Call{Context: env.Context, Config: env.Config}
+ cfg, err := connectorConfig(call)
+ if err != nil {
+ writeError(w, err)
+ return
+ }
+ ctx, cancel := callContext(r.Context(), env)
+ defer cancel()
+ s := newStream(w)
+ cursor, err := c.Fetch(ctx, call, cfg, in, s)
+ if err != nil {
+ s.fail(err)
+ return
+ }
+ s.end(cursor)
+ })
+}
diff --git a/pluginsdk/go.mod b/pluginsdk/go.mod
new file mode 100644
index 00000000..bb24df1b
--- /dev/null
+++ b/pluginsdk/go.mod
@@ -0,0 +1,3 @@
+module github.com/Tencent/WeKnora/pluginsdk
+
+go 1.24
diff --git a/pluginsdk/openapi_test.go b/pluginsdk/openapi_test.go
new file mode 100644
index 00000000..4d8a4346
--- /dev/null
+++ b/pluginsdk/openapi_test.go
@@ -0,0 +1,49 @@
+package pluginsdk
+
+import (
+ "os"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+)
+
+// The OpenAPI description and the SDK's routes must list the same endpoints,
+// so a protocol change cannot land in one and not the other.
+func TestOpenAPIMatchesRoutes(t *testing.T) {
+ raw, err := os.ReadFile("pluginapi/openapi.yaml")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Windows checkouts may carry CRLF line endings.
+ spec := strings.ReplaceAll(string(raw), "\r\n", "\n")
+ var documented []string
+ for _, m := range regexp.MustCompile(`(?m)^ (/v1/[^:]+):$`).FindAllStringSubmatch(spec, -1) {
+ documented = append(documented, m[1])
+ }
+ src, err := os.ReadFile("plugin.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, f := range []string{"websearch.go", "connector.go"} {
+ b, err := os.ReadFile(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ src = append(src, b...)
+ }
+ var routed []string
+ route := regexp.MustCompile(`HandleFunc\(\s*"(?:GET|POST) (/v1/[^"]+)"`)
+ for _, m := range route.FindAllStringSubmatch(string(src), -1) {
+ routed = append(routed, m[1])
+ }
+ sort.Strings(documented)
+ sort.Strings(routed)
+ if strings.Join(documented, "\n") != strings.Join(routed, "\n") {
+ t.Fatalf(
+ "openapi.yaml documents:\n%s\n\nthe SDK routes:\n%s",
+ strings.Join(documented, "\n"),
+ strings.Join(routed, "\n"),
+ )
+ }
+}
diff --git a/pluginsdk/plugin.go b/pluginsdk/plugin.go
new file mode 100644
index 00000000..61c04438
--- /dev/null
+++ b/pluginsdk/plugin.go
@@ -0,0 +1,349 @@
+// Package pluginsdk builds WeKnora code plugins in Go. A plugin registers what
+// it contributes and calls Serve; the SDK speaks the extension protocol
+// (package pluginapi): the handshake with the plugin host, authentication,
+// routing, streaming and error encoding.
+//
+// func main() {
+// p := pluginsdk.New(pluginsdk.Info{ID: "acme.rss", Version: "1.0.0"})
+// p.Connector("rss", &rssConnector{})
+// if err := p.Serve(); err != nil {
+// log.Fatal(err)
+// }
+// }
+//
+// Started by the WeKnora plugin host, the plugin listens where the host says
+// and accepts only the host's token. Run on its own (a remote plugin), it
+// listens on WEKNORA_PLUGIN_ADDR and verifies request signatures with
+// WEKNORA_PLUGIN_SECRET.
+package pluginsdk
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net"
+ "net/http"
+ "os"
+ "os/signal"
+ "sort"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// Info identifies the plugin; it must match plugin.yaml.
+type Info struct {
+ ID string
+ Version string
+}
+
+// Call is one request's context and configuration. Configuration secrets are
+// already decrypted; do not keep them beyond the call.
+type Call struct {
+ pluginapi.Context
+ Config pluginapi.Config
+}
+
+// DecodeInstance decodes the instance configuration into v.
+func (c *Call) DecodeInstance(v any) error {
+ return remarshal(c.Config.Instance, v)
+}
+
+// ConfigValidator checks the plugin's system and tenant configuration.
+type ConfigValidator func(ctx context.Context, call *Call) error
+
+// Plugin is a plugin under construction; register contributions, then Serve.
+type Plugin struct {
+ info Info
+ webSearch map[string]WebSearcher
+ connectors map[string]Connector
+ validate ConfigValidator
+ logger *slog.Logger
+ // ShutdownTimeout bounds how long Serve waits for calls in flight after
+ // a termination signal.
+ ShutdownTimeout time.Duration
+}
+
+// New starts a plugin.
+func New(info Info) *Plugin {
+ return &Plugin{
+ info: info,
+ webSearch: map[string]WebSearcher{},
+ connectors: map[string]Connector{},
+ logger: slog.New(slog.NewTextHandler(os.Stderr, nil)),
+ ShutdownTimeout: 60 * time.Second,
+ }
+}
+
+// Logger writes to stderr, which the plugin host forwards to WeKnora's logs.
+func (p *Plugin) Logger() *slog.Logger { return p.logger }
+
+// ValidateConfig registers the check behind POST /v1/config/validate.
+func (p *Plugin) ValidateConfig(fn ConfigValidator) { p.validate = fn }
+
+// Manifest is what the plugin reports at GET /v1/manifest.
+func (p *Plugin) Manifest() pluginapi.Manifest {
+ m := pluginapi.Manifest{
+ ID: p.info.ID, Version: p.info.Version, APIVersion: pluginapi.APIVersion,
+ Contributes: map[string][]string{},
+ }
+ add := func(point string, ids []string) {
+ if len(ids) > 0 {
+ sort.Strings(ids)
+ m.Contributes[point] = ids
+ }
+ }
+ add("webSearch", keys(p.webSearch))
+ add("connectors", keys(p.connectors))
+ return m
+}
+
+func keys[V any](m map[string]V) []string {
+ out := make([]string, 0, len(m))
+ for k := range m {
+ out = append(out, k)
+ }
+ return out
+}
+
+// Handler serves the protocol. Serve wraps it with authentication; tests
+// and custom servers can use it directly.
+func (p *Plugin) Handler() http.Handler {
+ mux := http.NewServeMux()
+ mux.HandleFunc("GET /v1/manifest", func(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, http.StatusOK, p.Manifest())
+ })
+ mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, http.StatusOK, pluginapi.Health{Status: "ok"})
+ })
+ mux.HandleFunc(
+ "POST /v1/config/validate",
+ p.unary(func(ctx context.Context, call *Call, _ json.RawMessage) (any, error) {
+ if p.validate != nil {
+ if err := p.validate(ctx, call); err != nil {
+ return nil, err
+ }
+ }
+ return struct{}{}, nil
+ }),
+ )
+ p.routeWebSearch(mux)
+ p.routeConnectors(mux)
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no endpoint %s %s", r.Method, r.URL.Path))
+ })
+ return recoverer(p.logger, mux)
+}
+
+// unaryFunc answers one call with one output.
+type unaryFunc func(ctx context.Context, call *Call, input json.RawMessage) (any, error)
+
+func (p *Plugin) unary(fn unaryFunc) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ env, err := readEnvelope(r)
+ if err != nil {
+ writeError(w, err)
+ return
+ }
+ ctx, cancel := callContext(r.Context(), env)
+ defer cancel()
+ out, err := fn(ctx, &Call{Context: env.Context, Config: env.Config}, env.Input)
+ if err != nil {
+ writeError(w, err)
+ return
+ }
+ raw, err := json.Marshal(out)
+ if err != nil {
+ writeError(w, err)
+ return
+ }
+ writeJSON(w, http.StatusOK, pluginapi.Output{Output: raw})
+ }
+}
+
+func readEnvelope(r *http.Request) (*pluginapi.Envelope, error) {
+ var env pluginapi.Envelope
+ if err := json.NewDecoder(r.Body).Decode(&env); err != nil {
+ return nil, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode envelope: %v", err)
+ }
+ return &env, nil
+}
+
+// callContext applies the envelope deadline.
+func callContext(parent context.Context, env *pluginapi.Envelope) (context.Context, context.CancelFunc) {
+ if env.Context.Deadline != nil {
+ return context.WithDeadline(parent, *env.Context.Deadline)
+ }
+ return context.WithCancel(parent)
+}
+
+func decodeInput[T any](raw json.RawMessage) (T, error) {
+ var v T
+ if len(raw) == 0 {
+ return v, nil
+ }
+ if err := json.Unmarshal(raw, &v); err != nil {
+ return v, pluginapi.Errorf(pluginapi.CodeBadRequest, "decode input: %v", err)
+ }
+ return v, nil
+}
+
+func remarshal(from, to any) error {
+ b, err := json.Marshal(from)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(b, to)
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _ = json.NewEncoder(w).Encode(v)
+}
+
+// toError maps any error to a protocol error.
+func toError(err error) *pluginapi.Error {
+ if e, ok := pluginapi.AsError(err); ok {
+ return e
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ return pluginapi.Errorf(pluginapi.CodeUnavailable, "deadline exceeded")
+ }
+ return &pluginapi.Error{Code: pluginapi.CodeInternal, Message: err.Error()}
+}
+
+func writeError(w http.ResponseWriter, err error) {
+ e := toError(err)
+ writeJSON(w, e.Code.HTTPStatus(), pluginapi.ErrorBody{Error: *e})
+}
+
+func recoverer(logger *slog.Logger, next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer func() {
+ if v := recover(); v != nil {
+ logger.Error("plugin panicked", "path", r.URL.Path, "panic", v)
+ writeError(w, fmt.Errorf("plugin panicked: %v", v))
+ }
+ }()
+ next.ServeHTTP(w, r)
+ })
+}
+
+// Serve runs the plugin until SIGTERM or SIGINT, then drains calls in
+// flight. Started by the plugin host it listens where the host says and
+// prints the handshake; otherwise it serves as a remote plugin.
+func (p *Plugin) Serve() error {
+ ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
+ defer stop()
+ return p.ServeContext(ctx)
+}
+
+// ServeContext is Serve with a caller-controlled lifetime.
+func (p *Plugin) ServeContext(ctx context.Context) error {
+ ln, auth, handshake, err := p.listen()
+ if err != nil {
+ return err
+ }
+ srv := &http.Server{Handler: auth(p.Handler()), ReadHeaderTimeout: 10 * time.Second}
+ errc := make(chan error, 1)
+ go func() { errc <- srv.Serve(ln) }()
+ if handshake != nil {
+ // The host waits for this exact line; everything else on stdout is
+ // treated as log output.
+ _, _ = fmt.Fprintln(os.Stdout, handshake.String())
+ } else {
+ p.logger.Info("plugin serving", "addr", ln.Addr().String())
+ }
+ select {
+ case err := <-errc:
+ return err
+ case <-ctx.Done():
+ }
+ shutdown, cancel := context.WithTimeout(context.Background(), p.ShutdownTimeout)
+ defer cancel()
+ return srv.Shutdown(shutdown)
+}
+
+type middleware func(http.Handler) http.Handler
+
+// listen picks host mode (socket + token from the environment) or remote
+// mode (address + signing secret).
+func (p *Plugin) listen() (net.Listener, middleware, *pluginapi.Handshake, error) {
+ if addr := os.Getenv(pluginapi.EnvSocket); addr != "" {
+ network := os.Getenv(pluginapi.EnvNetwork)
+ if network == "" {
+ network = "unix"
+ }
+ token := os.Getenv(pluginapi.EnvToken)
+ if token == "" {
+ return nil, nil, nil, fmt.Errorf("%s is set without %s", pluginapi.EnvSocket, pluginapi.EnvToken)
+ }
+ if network == "unix" {
+ _ = os.Remove(addr)
+ }
+ ln, err := net.Listen(network, addr)
+ if err != nil {
+ return nil, nil, nil, fmt.Errorf("listen %s %s: %w", network, addr, err)
+ }
+ hs := &pluginapi.Handshake{Protocol: pluginapi.ProtocolVersion, Network: network, Address: ln.Addr().String()}
+ return ln, bearerAuth(token), hs, nil
+ }
+ addr := os.Getenv(pluginapi.EnvAddr)
+ if addr == "" {
+ addr = ":8080"
+ }
+ secret := os.Getenv(pluginapi.EnvSecret)
+ if secret == "" {
+ return nil, nil, nil, fmt.Errorf(
+ "a remote plugin needs %s (the secret shown when it was registered)",
+ pluginapi.EnvSecret,
+ )
+ }
+ ln, err := net.Listen("tcp", addr)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ return ln, signatureAuth([]byte(secret)), nil, nil
+}
+
+func bearerAuth(token string) middleware {
+ want := "Bearer " + token
+ return func(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != want {
+ writeError(w, pluginapi.Errorf(pluginapi.CodeUnauthorized, "missing or wrong host token"))
+ return
+ }
+ next.ServeHTTP(w, r)
+ })
+ }
+}
+
+// maxBody bounds a signed request body read into memory for verification.
+const maxBody = 32 << 20
+
+func signatureAuth(secret []byte) middleware {
+ return func(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := io.ReadAll(io.LimitReader(r.Body, maxBody+1))
+ if err != nil || len(body) > maxBody {
+ writeError(w, pluginapi.Errorf(pluginapi.CodeBadRequest, "request body unreadable or too large"))
+ return
+ }
+ err = pluginapi.VerifySignature(secret, r.Header.Get(pluginapi.TimestampHeader),
+ r.Header.Get(pluginapi.SignatureHeader), body, time.Now())
+ if err != nil {
+ writeError(w, pluginapi.Errorf(pluginapi.CodeUnauthorized, "%v", err))
+ return
+ }
+ r.Body = io.NopCloser(strings.NewReader(string(body)))
+ next.ServeHTTP(w, r)
+ })
+ }
+}
diff --git a/pluginsdk/plugin_test.go b/pluginsdk/plugin_test.go
new file mode 100644
index 00000000..80b4b174
--- /dev/null
+++ b/pluginsdk/plugin_test.go
@@ -0,0 +1,259 @@
+package pluginsdk
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "fmt"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/pluginsdk/client"
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+type feedConnector struct{ items int }
+
+func (f *feedConnector) Validate(_ context.Context, _ *Call, cfg ConnectorConfig) error {
+ if cfg.Settings["url"] == nil {
+ return pluginapi.InvalidConfig("feed URL is required", map[string]string{"settings.url": "required"})
+ }
+ return nil
+}
+
+func (f *feedConnector) ListResources(
+ _ context.Context,
+ _ *Call,
+ cfg ConnectorConfig,
+ parentID string,
+) ([]pluginapi.Resource, error) {
+ if parentID != "" {
+ return nil, nil
+ }
+ return []pluginapi.Resource{{ExternalID: fmt.Sprint(cfg.Settings["url"]), Name: "Feed"}}, nil
+}
+
+func (f *feedConnector) Fetch(
+ _ context.Context, _ *Call, _ ConnectorConfig, in pluginapi.FetchInput, s *Stream,
+) (*pluginapi.Cursor, error) {
+ start := 0
+ if in.Cursor != nil {
+ start = int(in.Cursor.State["next"].(float64))
+ }
+ for i := start; i < f.items; i++ {
+ item := pluginapi.FetchedItem{ExternalID: fmt.Sprint(i), Title: fmt.Sprint("item ", i), Content: []byte("body")}
+ if err := s.Item(item); err != nil {
+ return nil, err
+ }
+ if err := s.Checkpoint(pluginapi.Cursor{State: map[string]any{"next": i + 1}}); err != nil {
+ return nil, err
+ }
+ if i == 2 && in.Mode == "fail-midway" {
+ return nil, pluginapi.Errorf(pluginapi.CodeUnavailable, "upstream went away")
+ }
+ }
+ return &pluginapi.Cursor{State: map[string]any{"next": f.items}}, nil
+}
+
+func testPlugin(t *testing.T) (*Plugin, *client.Client) {
+ t.Helper()
+ p := New(Info{ID: "acme.feed", Version: "1.0.0"})
+ p.Connector("feed", &feedConnector{items: 5})
+ p.WebSearch(
+ "echo",
+ WebSearchFunc(func(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error) {
+ switch in.Query {
+ case "panic":
+ panic("boom")
+ case "deadline":
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ }
+ return &pluginapi.SearchOutput{Results: []pluginapi.SearchResult{{
+ Title: in.Query,
+ URL: fmt.Sprint("https://example.com/?tenant=", call.TenantID,
+ "&key=", call.Config.Instance["api_key"]),
+ }}}, nil
+ }),
+ )
+ srv := httptest.NewServer(p.Handler())
+ t.Cleanup(srv.Close)
+ return p, client.New(srv.URL, nil, nil)
+}
+
+func TestUnaryCallsAndErrors(t *testing.T) {
+ ctx := context.Background()
+ _, c := testPlugin(t)
+
+ m, err := c.Manifest(ctx)
+ if err != nil || m.ID != "acme.feed" || m.Contributes["connectors"][0] != "feed" ||
+ m.APIVersion != pluginapi.APIVersion {
+ t.Fatalf("manifest = %+v, %v", m, err)
+ }
+ if err := c.Health(ctx); err != nil {
+ t.Fatal(err)
+ }
+
+ env := pluginapi.Envelope{
+ Context: pluginapi.Context{TenantID: 7},
+ Config: pluginapi.Config{Instance: map[string]any{"api_key": "k1"}},
+ }
+ var out pluginapi.SearchOutput
+ if err := c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "hi"}, &out); err != nil {
+ t.Fatal(err)
+ }
+ if len(out.Results) != 1 || out.Results[0].URL != "https://example.com/?tenant=7&key=k1" {
+ t.Fatalf("search = %+v", out)
+ }
+
+ err = c.Call(ctx, pluginapi.ConnectorValidatePath("feed"), pluginapi.Envelope{}, nil, nil)
+ if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeInvalidConfig ||
+ e.Details.Fields["settings.url"] != "required" {
+ t.Fatalf("validate = %v", err)
+ }
+ err = c.Call(ctx, pluginapi.SearchPath("nope"), env, pluginapi.SearchInput{}, nil)
+ if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeNotFound {
+ t.Fatalf("unknown provider = %v", err)
+ }
+ err = c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "panic"}, nil)
+ if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeInternal ||
+ !strings.Contains(e.Message, "boom") {
+ t.Fatalf("panic = %v", err)
+ }
+ past := time.Now().Add(-time.Second)
+ env.Context.Deadline = &past
+ err = c.Call(ctx, pluginapi.SearchPath("echo"), env, pluginapi.SearchInput{Query: "deadline"}, nil)
+ if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeUnavailable {
+ t.Fatalf("the envelope deadline must reach the handler's context, got %v", err)
+ }
+}
+
+func TestStreamingFetch(t *testing.T) {
+ ctx := context.Background()
+ _, c := testPlugin(t)
+ env := pluginapi.Envelope{
+ Config: pluginapi.Config{Instance: map[string]any{"settings": map[string]any{"url": "u"}}},
+ }
+
+ var items []string
+ var last *pluginapi.Cursor
+ end, err := c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env,
+ pluginapi.FetchInput{Mode: pluginapi.FetchFull, Cursor: &pluginapi.Cursor{State: map[string]any{"next": 3}}},
+ func(ev pluginapi.Event) error {
+ switch ev.Type {
+ case pluginapi.EventItem:
+ var it pluginapi.FetchedItem
+ if err := remarshal(ev.Data, &it); err != nil {
+ return err
+ }
+ items = append(items, it.ExternalID+":"+string(it.Content))
+ case pluginapi.EventCheckpoint:
+ last = &pluginapi.Cursor{}
+ return remarshal(ev.Data, last)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Join(items, ",") != "3:body,4:body" || last.State["next"].(float64) != 5 ||
+ !strings.Contains(string(end), `"next":5`) {
+ t.Fatalf("items=%v last=%+v end=%s", items, last, end)
+ }
+
+ _, err = c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env, pluginapi.FetchInput{Mode: "fail-midway"},
+ func(pluginapi.Event) error { return nil })
+ if e, ok := pluginapi.AsError(err); !ok || e.Code != pluginapi.CodeUnavailable || !e.Retryable {
+ t.Fatalf("mid-stream failure = %v", err)
+ }
+
+ stop := errors.New("ingest failed")
+ _, err = c.Stream(ctx, pluginapi.ConnectorFetchPath("feed"), env, pluginapi.FetchInput{},
+ func(pluginapi.Event) error { return stop })
+ if !errors.Is(err, stop) {
+ t.Fatalf("a handler error must abort the stream, got %v", err)
+ }
+}
+
+func TestAuthModes(t *testing.T) {
+ ctx := context.Background()
+ p, _ := testPlugin(t)
+
+ bearer := httptest.NewServer(bearerAuth("tok")(p.Handler()))
+ defer bearer.Close()
+ if err := client.New(bearer.URL, nil, client.Bearer("wrong")).Health(ctx); !isCode(
+ err,
+ pluginapi.CodeUnauthorized,
+ ) {
+ t.Fatalf("wrong token = %v", err)
+ }
+ if err := client.New(bearer.URL, nil, client.Bearer("tok")).Health(ctx); err != nil {
+ t.Fatal(err)
+ }
+
+ signed := httptest.NewServer(signatureAuth([]byte("s3cret"))(p.Handler()))
+ defer signed.Close()
+ if err := client.New(signed.URL, nil, client.Signed("other")).Health(ctx); !isCode(
+ err,
+ pluginapi.CodeUnauthorized,
+ ) {
+ t.Fatalf("wrong secret = %v", err)
+ }
+ var out pluginapi.SearchOutput
+ err := client.New(signed.URL, nil, client.Signed("s3cret")).
+ Call(ctx, pluginapi.SearchPath("echo"), pluginapi.Envelope{}, pluginapi.SearchInput{Query: "q"}, &out)
+ if err != nil || len(out.Results) != 1 {
+ t.Fatalf("signed call = %+v, %v", out, err)
+ }
+}
+
+func isCode(err error, code pluginapi.ErrorCode) bool {
+ e, ok := pluginapi.AsError(err)
+ return ok && e.Code == code
+}
+
+// ServeContext in host mode listens on the socket from the environment,
+// prints the handshake and requires the token.
+func TestServeHostHandshake(t *testing.T) {
+ dir, err := os.MkdirTemp("", "wkp")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = os.RemoveAll(dir) }()
+ sock := filepath.Join(dir, "p.sock")
+ t.Setenv(pluginapi.EnvSocket, sock)
+ t.Setenv(pluginapi.EnvToken, "tok")
+
+ r, w, _ := os.Pipe()
+ stdout := os.Stdout
+ os.Stdout = w
+ defer func() { os.Stdout = stdout }()
+
+ p, _ := testPlugin(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan error, 1)
+ go func() { done <- p.ServeContext(ctx) }()
+
+ line, err := bufio.NewReader(r).ReadString('\n')
+ if err != nil {
+ t.Fatal(err)
+ }
+ os.Stdout = stdout
+ hs, ok, err := pluginapi.ParseHandshake(line)
+ if !ok || err != nil || hs.Network != "unix" || hs.Address != sock {
+ t.Fatalf("handshake %q = %+v %v %v", line, hs, ok, err)
+ }
+ c := client.ForHandshake(hs, client.Bearer("tok"))
+ if err := c.Health(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ cancel()
+ if err := <-done; err != nil {
+ t.Fatalf("shutdown: %v", err)
+ }
+}
diff --git a/pluginsdk/pluginapi/connector.go b/pluginsdk/pluginapi/connector.go
new file mode 100644
index 00000000..07cbc9fd
--- /dev/null
+++ b/pluginsdk/pluginapi/connector.go
@@ -0,0 +1,96 @@
+package pluginapi
+
+import "time"
+
+// ConnectorValidatePath is the validate endpoint of connector id.
+func ConnectorValidatePath(id string) string { return "/v1/connectors/" + id + "/validate" }
+
+// ConnectorListResourcesPath is the list-resources endpoint of connector id.
+func ConnectorListResourcesPath(id string) string { return "/v1/connectors/" + id + "/list-resources" }
+
+// ConnectorResolveAncestorsPath is the resolve-ancestors endpoint of connector id.
+func ConnectorResolveAncestorsPath(id string) string {
+ return "/v1/connectors/" + id + "/resolve-ancestors"
+}
+
+// ConnectorFetchPath is the streaming fetch endpoint of connector id.
+func ConnectorFetchPath(id string) string { return "/v1/connectors/" + id + "/fetch" }
+
+// The connector's instance configuration arrives in Envelope.Config.Instance
+// as {"credentials": {...}, "settings": {...}, "resourceIds": [...]}.
+
+// ListResourcesInput asks for the resources under ParentID ("" = top level).
+type ListResourcesInput struct {
+ ParentID string `json:"parentId,omitempty"`
+}
+
+// ListResourcesOutput lists resources a user can pick for syncing.
+type ListResourcesOutput struct {
+ Resources []Resource `json:"resources"`
+}
+
+// Resource is something a data source can sync (a space, folder, feed).
+type Resource struct {
+ ExternalID string `json:"externalId"`
+ Name string `json:"name"`
+ Type string `json:"type,omitempty"`
+ Description string `json:"description,omitempty"`
+ URL string `json:"url,omitempty"`
+ ModifiedAt *time.Time `json:"modifiedAt,omitempty"`
+ ParentID string `json:"parentId,omitempty"`
+ HasChildren bool `json:"hasChildren,omitempty"`
+ Metadata map[string]any `json:"metadata,omitempty"`
+}
+
+// ResolveAncestorsInput asks which ancestors of the given resources a
+// lazily loaded picker must expand.
+type ResolveAncestorsInput struct {
+ ResourceIDs []string `json:"resourceIds"`
+}
+
+// ResolveAncestorsOutput is the deduplicated set of ancestor IDs.
+type ResolveAncestorsOutput struct {
+ Ancestors []string `json:"ancestors"`
+}
+
+// Fetch modes.
+const (
+ FetchFull = "full"
+ FetchIncremental = "incremental"
+)
+
+// FetchInput starts or resumes a sync. The answer is a stream of "item"
+// events (FetchedItem), "checkpoint" events (Cursor) at page boundaries and
+// an "end" event carrying the final Cursor.
+type FetchInput struct {
+ Mode string `json:"mode"`
+ // Cursor is the last checkpoint or final cursor; nil starts from scratch.
+ Cursor *Cursor `json:"cursor,omitempty"`
+ // ResourceIDs are the selected resources (also in config.instance).
+ ResourceIDs []string `json:"resourceIds,omitempty"`
+}
+
+// Cursor is the connector's resumable state. WeKnora stores it and hands it
+// back; its content is the plugin's business. A checkpoint must be a
+// complete snapshot: resuming from it reproduces all progress so far.
+type Cursor struct {
+ LastSyncTime *time.Time `json:"lastSyncTime,omitempty"`
+ State map[string]any `json:"state,omitempty"`
+}
+
+// FetchedItem is one document fetched from the source.
+type FetchedItem struct {
+ ExternalID string `json:"externalId"`
+ Title string `json:"title"`
+ // Content is the document body; Markdown preferred. JSON carries it as
+ // base64.
+ Content []byte `json:"content,omitempty"`
+ ContentType string `json:"contentType,omitempty"`
+ FileName string `json:"fileName,omitempty"`
+ URL string `json:"url,omitempty"`
+ UpdatedAt *time.Time `json:"updatedAt,omitempty"`
+ CreatedAt *time.Time `json:"createdAt,omitempty"`
+ Metadata map[string]string `json:"metadata,omitempty"`
+ IsDeleted bool `json:"isDeleted,omitempty"`
+ SourceResourceID string `json:"sourceResourceId,omitempty"`
+}
diff --git a/pluginsdk/pluginapi/doc.go b/pluginsdk/pluginapi/doc.go
new file mode 100644
index 00000000..f22145c4
--- /dev/null
+++ b/pluginsdk/pluginapi/doc.go
@@ -0,0 +1,32 @@
+// Package pluginapi is version 1 of the WeKnora extension protocol: the
+// messages WeKnora and a code plugin exchange over HTTP. It has no
+// dependencies so that plugins and WeKnora share one definition; openapi.yaml
+// in this directory describes the same protocol for other languages.
+//
+// Every call is a POST whose body is an Envelope and whose successful answer
+// is {"output": ...}. Streaming calls answer with NDJSON Events instead. A
+// failed call answers with a non-2xx status and an ErrorBody. Within a major
+// version the protocol only grows: new optional fields and new endpoints.
+// Both sides ignore fields they do not know.
+package pluginapi
+
+// ProtocolVersion is the major version this package speaks; it travels in
+// the ProtocolHeader of every request.
+const ProtocolVersion = "1"
+
+// APIVersion is the manifest apiVersion of plugins speaking this protocol.
+const APIVersion = "weknora.plugin/v1"
+
+// Request headers.
+const (
+ ProtocolHeader = "X-WeKnora-Protocol"
+ RequestIDHeader = "X-Request-Id"
+ // SignatureHeader and TimestampHeader authenticate calls to remote
+ // plugins: hex HMAC-SHA256 over "." with the shared
+ // secret. Host plugins get a bearer token instead.
+ SignatureHeader = "X-WeKnora-Signature"
+ TimestampHeader = "X-WeKnora-Timestamp"
+)
+
+// NDJSONContentType is the content type of streaming answers.
+const NDJSONContentType = "application/x-ndjson"
diff --git a/pluginsdk/pluginapi/envelope.go b/pluginsdk/pluginapi/envelope.go
new file mode 100644
index 00000000..68356297
--- /dev/null
+++ b/pluginsdk/pluginapi/envelope.go
@@ -0,0 +1,71 @@
+package pluginapi
+
+import (
+ "encoding/json"
+ "time"
+)
+
+// Envelope is the body of every call. Plugins must be stateless: everything
+// about the tenant comes in the envelope, and credentials must not be cached
+// across calls.
+type Envelope struct {
+ Context Context `json:"context"`
+ Config Config `json:"config"`
+ // Input is the endpoint-specific request, such as a SearchInput.
+ Input json.RawMessage `json:"input,omitempty"`
+}
+
+// Context describes who is calling and until when.
+type Context struct {
+ TenantID uint64 `json:"tenantId"`
+ UserID string `json:"userId,omitempty"`
+ Locale string `json:"locale,omitempty"`
+ RequestID string `json:"requestId,omitempty"`
+ // Deadline is when WeKnora stops waiting; plugins should give up by then.
+ Deadline *time.Time `json:"deadline,omitempty"`
+ // Host lets the plugin call back into WeKnora (Host API); absent when
+ // the plugin was granted no scopes.
+ Host *HostAccess `json:"host,omitempty"`
+}
+
+// HostAccess is where and how a plugin calls the WeKnora Host API.
+type HostAccess struct {
+ URL string `json:"url"`
+ // Token is a short-lived bearer token bound to this tenant and the
+ // plugin's granted scopes.
+ Token string `json:"token"`
+}
+
+// Config carries the three configuration scopes separately, secrets already
+// decrypted. They are never merged, so a key means the same thing wherever
+// it appears.
+type Config struct {
+ // System is the platform-wide configuration (config.system).
+ System map[string]any `json:"system,omitempty"`
+ // Tenant is the workspace configuration (config.tenant).
+ Tenant map[string]any `json:"tenant,omitempty"`
+ // Instance is the configuration of the integration instance being used:
+ // a data source, a web search provider.
+ Instance map[string]any `json:"instance,omitempty"`
+}
+
+// Output wraps a successful answer.
+type Output struct {
+ Output json.RawMessage `json:"output"`
+}
+
+// Manifest is what GET /v1/manifest answers: enough for WeKnora to check that
+// the process it started is the package it installed.
+type Manifest struct {
+ ID string `json:"id"`
+ Version string `json:"version"`
+ APIVersion string `json:"apiVersion"`
+ // Contributes lists the contribution IDs the process serves per
+ // extension point, e.g. {"connectors": ["rss"]}.
+ Contributes map[string][]string `json:"contributes"`
+}
+
+// Health is what GET /v1/health answers.
+type Health struct {
+ Status string `json:"status"` // "ok"
+}
diff --git a/pluginsdk/pluginapi/errors.go b/pluginsdk/pluginapi/errors.go
new file mode 100644
index 00000000..d525a769
--- /dev/null
+++ b/pluginsdk/pluginapi/errors.go
@@ -0,0 +1,90 @@
+package pluginapi
+
+import (
+ "errors"
+ "fmt"
+)
+
+// ErrorCode classifies a failed call; WeKnora reacts to each differently.
+type ErrorCode string
+
+const (
+ // CodeInvalidConfig means the configuration is wrong; Details.Fields says
+ // where. WeKnora shows it on the form.
+ CodeInvalidConfig ErrorCode = "invalid_config"
+ // CodeUnauthorized means the credentials no longer work.
+ CodeUnauthorized ErrorCode = "unauthorized"
+ // CodeNotFound means the resource does not exist.
+ CodeNotFound ErrorCode = "not_found"
+ // CodeRateLimited means back off for Details.RetryAfter seconds.
+ CodeRateLimited ErrorCode = "rate_limited"
+ // CodeUnavailable means the upstream service is down; retry later.
+ CodeUnavailable ErrorCode = "unavailable"
+ // CodeBadRequest means the call itself is malformed (protocol error).
+ CodeBadRequest ErrorCode = "bad_request"
+ // CodeInternal means the plugin failed.
+ CodeInternal ErrorCode = "internal"
+)
+
+// Error is a failed call. Plugins return it (or wrap it) to choose the code;
+// any other error becomes CodeInternal.
+type Error struct {
+ Code ErrorCode `json:"code"`
+ Message string `json:"message"`
+ Retryable bool `json:"retryable,omitempty"`
+ Details *ErrorDetails `json:"details,omitempty"`
+}
+
+// ErrorDetails refines some codes.
+type ErrorDetails struct {
+ // Fields are per-field problems for CodeInvalidConfig, keyed by the
+ // dotted field path.
+ Fields map[string]string `json:"fields,omitempty"`
+ // RetryAfter is seconds to wait for CodeRateLimited.
+ RetryAfter int `json:"retryAfter,omitempty"`
+}
+
+// ErrorBody is the JSON body of a failed call.
+type ErrorBody struct {
+ Error Error `json:"error"`
+}
+
+func (e *Error) Error() string { return fmt.Sprintf("%s: %s", e.Code, e.Message) }
+
+// Errorf builds an Error. Unavailable and rate-limited errors are retryable.
+func Errorf(code ErrorCode, format string, args ...any) *Error {
+ return &Error{
+ Code: code, Message: fmt.Sprintf(format, args...),
+ Retryable: code == CodeUnavailable || code == CodeRateLimited,
+ }
+}
+
+// InvalidConfig reports field problems, keyed by dotted path.
+func InvalidConfig(message string, fields map[string]string) *Error {
+ return &Error{Code: CodeInvalidConfig, Message: message, Details: &ErrorDetails{Fields: fields}}
+}
+
+// AsError finds an *Error in err's chain.
+func AsError(err error) (*Error, bool) {
+ var e *Error
+ ok := errors.As(err, &e)
+ return e, ok
+}
+
+// HTTPStatus is the status an error code travels with.
+func (c ErrorCode) HTTPStatus() int {
+ switch c {
+ case CodeInvalidConfig, CodeBadRequest:
+ return 400
+ case CodeUnauthorized:
+ return 401
+ case CodeNotFound:
+ return 404
+ case CodeRateLimited:
+ return 429
+ case CodeUnavailable:
+ return 503
+ default:
+ return 500
+ }
+}
diff --git a/pluginsdk/pluginapi/handshake.go b/pluginsdk/pluginapi/handshake.go
new file mode 100644
index 00000000..e2ce7930
--- /dev/null
+++ b/pluginsdk/pluginapi/handshake.go
@@ -0,0 +1,63 @@
+package pluginapi
+
+import (
+ "fmt"
+ "strings"
+)
+
+// Environment a host plugin is started with.
+const (
+ // EnvSocket is where the plugin must listen (a unix socket path, or a
+ // host:port when EnvNetwork is "tcp").
+ EnvSocket = "WEKNORA_PLUGIN_SOCKET"
+ // EnvNetwork is "unix" (default) or "tcp".
+ EnvNetwork = "WEKNORA_PLUGIN_NETWORK"
+ // EnvToken is the bearer token every call from the host carries.
+ EnvToken = "WEKNORA_PLUGIN_TOKEN"
+ // EnvPluginID and EnvPluginVersion name the package being run.
+ EnvPluginID = "WEKNORA_PLUGIN_ID"
+ EnvPluginVersion = "WEKNORA_PLUGIN_VERSION"
+ // EnvAddr and EnvSecret configure a remote plugin (not started by a
+ // host): where to listen and the shared secret calls are signed with.
+ EnvAddr = "WEKNORA_PLUGIN_ADDR"
+ EnvSecret = "WEKNORA_PLUGIN_SECRET"
+)
+
+// handshakePrefix starts the line a host plugin prints on stdout once it is
+// ready: WEKNORA_PLUGIN|1|unix|/path/to.sock
+const handshakePrefix = "WEKNORA_PLUGIN"
+
+// Handshake is the readiness line of a host plugin.
+type Handshake struct {
+ Protocol string
+ Network string // unix | tcp
+ Address string
+}
+
+// String renders the handshake line (without a newline).
+func (h Handshake) String() string {
+ return strings.Join([]string{handshakePrefix, h.Protocol, h.Network, h.Address}, "|")
+}
+
+// ParseHandshake reads a handshake line. ok is false for any other line,
+// which the host treats as log output.
+func ParseHandshake(line string) (h Handshake, ok bool, err error) {
+ parts := strings.Split(strings.TrimSpace(line), "|")
+ if len(parts) == 0 || parts[0] != handshakePrefix {
+ return Handshake{}, false, nil
+ }
+ if len(parts) != 4 {
+ return Handshake{}, true, fmt.Errorf("malformed handshake %q", line)
+ }
+ h = Handshake{Protocol: parts[1], Network: parts[2], Address: parts[3]}
+ if h.Protocol != ProtocolVersion {
+ return h, true, fmt.Errorf("plugin speaks protocol %s, host speaks %s", h.Protocol, ProtocolVersion)
+ }
+ if h.Network != "unix" && h.Network != "tcp" {
+ return h, true, fmt.Errorf("unknown handshake network %q", h.Network)
+ }
+ if h.Address == "" {
+ return h, true, fmt.Errorf("handshake has no address")
+ }
+ return h, true, nil
+}
diff --git a/pluginsdk/pluginapi/openapi.yaml b/pluginsdk/pluginapi/openapi.yaml
new file mode 100644
index 00000000..711664c3
--- /dev/null
+++ b/pluginsdk/pluginapi/openapi.yaml
@@ -0,0 +1,324 @@
+openapi: 3.1.0
+info:
+ title: WeKnora extension protocol
+ version: "1"
+ description: |
+ What WeKnora calls on a code plugin. Every POST takes an Envelope and
+ answers {"output": ...}, or streams NDJSON Events. Failures answer a
+ non-2xx status with an ErrorBody. Within a major version the protocol
+ only grows; both sides ignore fields they do not know.
+
+ Host plugins (started by the WeKnora plugin host) are called over the
+ socket from their handshake with `Authorization: Bearer `. Remote
+ plugins are called over HTTPS and verify `X-WeKnora-Signature`:
+ hex HMAC-SHA256 of "." with the shared secret,
+ rejecting timestamps more than 5 minutes off.
+servers:
+ - url: http://plugin
+security:
+ - hostToken: []
+ - signature: []
+paths:
+ /v1/manifest:
+ get:
+ summary: What this process is and serves
+ responses:
+ "200":
+ description: The manifest
+ content:
+ application/json:
+ schema: { $ref: "#/components/schemas/Manifest" }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/health:
+ get:
+ summary: Liveness
+ responses:
+ "200":
+ description: Healthy
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [status]
+ properties: { status: { const: ok } }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/config/validate:
+ post:
+ summary: Check the plugin's system and tenant configuration
+ requestBody: { $ref: "#/components/requestBodies/Envelope" }
+ responses:
+ "200": { $ref: "#/components/responses/EmptyOutput" }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/websearch/{id}/search:
+ post:
+ summary: Web search (WebSearchProvider.Search)
+ parameters: [{ $ref: "#/components/parameters/ContributionID" }]
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ allOf:
+ - $ref: "#/components/schemas/Envelope"
+ - properties: { input: { $ref: "#/components/schemas/SearchInput" } }
+ responses:
+ "200":
+ description: Results
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [output]
+ properties: { output: { $ref: "#/components/schemas/SearchOutput" } }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/connectors/{id}/validate:
+ post:
+ summary: Check a data source configuration (Connector.Validate)
+ parameters: [{ $ref: "#/components/parameters/ContributionID" }]
+ requestBody: { $ref: "#/components/requestBodies/Envelope" }
+ responses:
+ "200": { $ref: "#/components/responses/EmptyOutput" }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/connectors/{id}/list-resources:
+ post:
+ summary: List syncable resources under a parent (Connector.ListResources)
+ parameters: [{ $ref: "#/components/parameters/ContributionID" }]
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ allOf:
+ - $ref: "#/components/schemas/Envelope"
+ - properties:
+ input:
+ type: object
+ properties: { parentId: { type: string } }
+ responses:
+ "200":
+ description: Resources
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [output]
+ properties:
+ output:
+ type: object
+ required: [resources]
+ properties:
+ resources: { type: array, items: { $ref: "#/components/schemas/Resource" } }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/connectors/{id}/resolve-ancestors:
+ post:
+ summary: Ancestors a lazy picker must expand to show a selection
+ parameters: [{ $ref: "#/components/parameters/ContributionID" }]
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ allOf:
+ - $ref: "#/components/schemas/Envelope"
+ - properties:
+ input:
+ type: object
+ properties: { resourceIds: { type: array, items: { type: string } } }
+ responses:
+ "200":
+ description: Ancestor IDs
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [output]
+ properties:
+ output:
+ type: object
+ required: [ancestors]
+ properties: { ancestors: { type: array, items: { type: string } } }
+ default: { $ref: "#/components/responses/Error" }
+ /v1/connectors/{id}/fetch:
+ post:
+ summary: Stream a full or incremental sync
+ description: |
+ Answers NDJSON: "item" events carry a FetchedItem, "checkpoint"
+ events a Cursor (a complete, resumable snapshot), and the stream
+ ends with "end" (data = final Cursor) or "error". A stream that
+ stops without either is treated as interrupted.
+ parameters: [{ $ref: "#/components/parameters/ContributionID" }]
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ allOf:
+ - $ref: "#/components/schemas/Envelope"
+ - properties: { input: { $ref: "#/components/schemas/FetchInput" } }
+ responses:
+ "200":
+ description: Event stream
+ content:
+ application/x-ndjson:
+ schema: { $ref: "#/components/schemas/Event" }
+ default: { $ref: "#/components/responses/Error" }
+components:
+ securitySchemes:
+ hostToken: { type: http, scheme: bearer }
+ signature: { type: apiKey, in: header, name: X-WeKnora-Signature }
+ parameters:
+ ContributionID:
+ name: id
+ in: path
+ required: true
+ description: The contribution's local ID from plugin.yaml
+ schema: { type: string }
+ requestBodies:
+ Envelope:
+ required: true
+ content:
+ application/json:
+ schema: { $ref: "#/components/schemas/Envelope" }
+ responses:
+ EmptyOutput:
+ description: Accepted
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [output]
+ properties: { output: { type: object } }
+ Error:
+ description: A failed call
+ content:
+ application/json:
+ schema: { $ref: "#/components/schemas/ErrorBody" }
+ schemas:
+ Envelope:
+ type: object
+ required: [context, config]
+ properties:
+ context:
+ type: object
+ required: [tenantId]
+ properties:
+ tenantId: { type: integer }
+ userId: { type: string }
+ locale: { type: string }
+ requestId: { type: string }
+ deadline: { type: string, format: date-time }
+ host:
+ type: object
+ description: Host API access; absent without granted scopes
+ properties:
+ url: { type: string }
+ token: { type: string }
+ config:
+ type: object
+ description: Configuration scopes, secrets decrypted, never merged
+ properties:
+ system: { type: object }
+ tenant: { type: object }
+ instance: { type: object }
+ input: {}
+ Manifest:
+ type: object
+ required: [id, version, apiVersion, contributes]
+ properties:
+ id: { type: string }
+ version: { type: string }
+ apiVersion: { const: weknora.plugin/v1 }
+ contributes:
+ type: object
+ additionalProperties: { type: array, items: { type: string } }
+ SearchInput:
+ type: object
+ required: [query]
+ properties:
+ query: { type: string }
+ maxResults: { type: integer }
+ includeDate: { type: boolean }
+ region: { type: string }
+ freshness: { type: string }
+ SearchOutput:
+ type: object
+ required: [results]
+ properties:
+ results:
+ type: array
+ items:
+ type: object
+ required: [title, url]
+ properties:
+ title: { type: string }
+ url: { type: string }
+ snippet: { type: string }
+ content: { type: string }
+ age: { type: string }
+ publishedAt: { type: string, format: date-time }
+ Resource:
+ type: object
+ required: [externalId, name]
+ properties:
+ externalId: { type: string }
+ name: { type: string }
+ type: { type: string }
+ description: { type: string }
+ url: { type: string }
+ modifiedAt: { type: string, format: date-time }
+ parentId: { type: string }
+ hasChildren: { type: boolean }
+ metadata: { type: object }
+ FetchInput:
+ type: object
+ required: [mode]
+ properties:
+ mode: { enum: [full, incremental] }
+ cursor: { $ref: "#/components/schemas/Cursor" }
+ resourceIds: { type: array, items: { type: string } }
+ Cursor:
+ type: object
+ properties:
+ lastSyncTime: { type: string, format: date-time }
+ state: { type: object }
+ FetchedItem:
+ type: object
+ required: [externalId, title]
+ properties:
+ externalId: { type: string }
+ title: { type: string }
+ content: { type: string, contentEncoding: base64 }
+ contentType: { type: string }
+ fileName: { type: string }
+ url: { type: string }
+ updatedAt: { type: string, format: date-time }
+ createdAt: { type: string, format: date-time }
+ metadata: { type: object, additionalProperties: { type: string } }
+ isDeleted: { type: boolean }
+ sourceResourceId: { type: string }
+ Event:
+ type: object
+ required: [type]
+ properties:
+ type: { enum: [item, checkpoint, progress, log, error, end] }
+ data: {}
+ error: { $ref: "#/components/schemas/Error" }
+ message: { type: string }
+ level: { enum: [debug, info, warn, error] }
+ Error:
+ type: object
+ required: [code, message]
+ properties:
+ code: { enum: [invalid_config, unauthorized, not_found, rate_limited, unavailable, bad_request, internal] }
+ message: { type: string }
+ retryable: { type: boolean }
+ details:
+ type: object
+ properties:
+ fields: { type: object, additionalProperties: { type: string } }
+ retryAfter: { type: integer }
+ ErrorBody:
+ type: object
+ required: [error]
+ properties:
+ error: { $ref: "#/components/schemas/Error" }
diff --git a/pluginsdk/pluginapi/pluginapi_test.go b/pluginsdk/pluginapi/pluginapi_test.go
new file mode 100644
index 00000000..f7b5b1cc
--- /dev/null
+++ b/pluginsdk/pluginapi/pluginapi_test.go
@@ -0,0 +1,48 @@
+package pluginapi
+
+import (
+ "strconv"
+ "testing"
+ "time"
+)
+
+func TestHandshake(t *testing.T) {
+ h := Handshake{Protocol: ProtocolVersion, Network: "unix", Address: "/tmp/p.sock"}
+ got, ok, err := ParseHandshake(h.String() + "\n")
+ if !ok || err != nil || got != h {
+ t.Fatalf("round trip = %+v %v %v", got, ok, err)
+ }
+ if _, ok, _ := ParseHandshake("starting up..."); ok {
+ t.Fatal("other output is not a handshake")
+ }
+ bad := []string{
+ "WEKNORA_PLUGIN|2|unix|/x", "WEKNORA_PLUGIN|1|pipe|/x", "WEKNORA_PLUGIN|1|unix|", "WEKNORA_PLUGIN|1",
+ }
+ for _, bad := range bad {
+ if _, ok, err := ParseHandshake(bad); !ok || err == nil {
+ t.Errorf("%q must be a rejected handshake", bad)
+ }
+ }
+}
+
+func TestSignature(t *testing.T) {
+ secret, body, now := []byte("k"), []byte(`{"a":1}`), time.Now()
+ ts := strconv.FormatInt(now.Unix(), 10)
+ sig := Sign(secret, now.Unix(), body)
+ if err := VerifySignature(secret, ts, sig, body, now); err != nil {
+ t.Fatal(err)
+ }
+ if VerifySignature(secret, ts, sig, []byte(`{"a":2}`), now) == nil {
+ t.Fatal("a changed body must fail")
+ }
+ if VerifySignature(secret, ts, sig, body, now.Add(10*time.Minute)) == nil {
+ t.Fatal("a replayed old request must fail")
+ }
+}
+
+func TestErrorStatus(t *testing.T) {
+ if Errorf(CodeRateLimited, "x").Code.HTTPStatus() != 429 || !Errorf(CodeUnavailable, "x").Retryable ||
+ Errorf(CodeInternal, "x").Retryable {
+ t.Fatal("error codes map wrong")
+ }
+}
diff --git a/pluginsdk/pluginapi/signature.go b/pluginsdk/pluginapi/signature.go
new file mode 100644
index 00000000..32d54783
--- /dev/null
+++ b/pluginsdk/pluginapi/signature.go
@@ -0,0 +1,37 @@
+package pluginapi
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "strconv"
+ "time"
+)
+
+// MaxClockSkew bounds how old a signed request may be.
+const MaxClockSkew = 5 * time.Minute
+
+// Sign returns the signature of a request body at a unix timestamp.
+func Sign(secret []byte, timestamp int64, body []byte) string {
+ mac := hmac.New(sha256.New, secret)
+ _, _ = fmt.Fprintf(mac, "%d.", timestamp)
+ _, _ = mac.Write(body)
+ return hex.EncodeToString(mac.Sum(nil))
+}
+
+// VerifySignature checks a signed request against the secret and clock.
+func VerifySignature(secret []byte, timestamp, signature string, body []byte, now time.Time) error {
+ ts, err := strconv.ParseInt(timestamp, 10, 64)
+ if err != nil {
+ return fmt.Errorf("bad %s", TimestampHeader)
+ }
+ if d := now.Sub(time.Unix(ts, 0)); d > MaxClockSkew || d < -MaxClockSkew {
+ return fmt.Errorf("request timestamp is outside the allowed clock skew")
+ }
+ want := Sign(secret, ts, body)
+ if !hmac.Equal([]byte(want), []byte(signature)) {
+ return fmt.Errorf("bad signature")
+ }
+ return nil
+}
diff --git a/pluginsdk/pluginapi/stream.go b/pluginsdk/pluginapi/stream.go
new file mode 100644
index 00000000..7f42bc09
--- /dev/null
+++ b/pluginsdk/pluginapi/stream.go
@@ -0,0 +1,34 @@
+package pluginapi
+
+import "encoding/json"
+
+// EventType is the kind of one NDJSON line in a streaming answer.
+type EventType string
+
+const (
+ // EventItem carries one result (a FetchedItem for connectors).
+ EventItem EventType = "item"
+ // EventCheckpoint carries a resumable cursor; WeKnora persists it.
+ EventCheckpoint EventType = "checkpoint"
+ // EventProgress reports progress for display.
+ EventProgress EventType = "progress"
+ // EventLog forwards a log line to WeKnora's logs.
+ EventLog EventType = "log"
+ // EventError ends the stream with an error.
+ EventError EventType = "error"
+ // EventEnd ends the stream successfully, with the final Data.
+ EventEnd EventType = "end"
+)
+
+// Event is one line of a streaming answer. A stream that stops without an
+// "end" or "error" event was interrupted and is treated as failed.
+type Event struct {
+ Type EventType `json:"type"`
+ Data json.RawMessage `json:"data,omitempty"`
+ // Error is set on EventError.
+ Error *Error `json:"error,omitempty"`
+ // Message is set on EventLog and EventProgress.
+ Message string `json:"message,omitempty"`
+ // Level is set on EventLog: debug, info, warn, error.
+ Level string `json:"level,omitempty"`
+}
diff --git a/pluginsdk/pluginapi/websearch.go b/pluginsdk/pluginapi/websearch.go
new file mode 100644
index 00000000..57b3601c
--- /dev/null
+++ b/pluginsdk/pluginapi/websearch.go
@@ -0,0 +1,32 @@
+package pluginapi
+
+import "time"
+
+// SearchPath is the web search endpoint for provider id.
+func SearchPath(id string) string { return "/v1/websearch/" + id + "/search" }
+
+// SearchInput is the input of a web search.
+type SearchInput struct {
+ Query string `json:"query"`
+ MaxResults int `json:"maxResults"`
+ IncludeDate bool `json:"includeDate,omitempty"`
+ // Region and Freshness are optional filters; a provider that cannot
+ // honour a requested filter must fail rather than ignore it.
+ Region string `json:"region,omitempty"`
+ Freshness string `json:"freshness,omitempty"`
+}
+
+// SearchOutput is the output of a web search.
+type SearchOutput struct {
+ Results []SearchResult `json:"results"`
+}
+
+// SearchResult is one hit.
+type SearchResult struct {
+ Title string `json:"title"`
+ URL string `json:"url"`
+ Snippet string `json:"snippet,omitempty"`
+ Content string `json:"content,omitempty"`
+ Age string `json:"age,omitempty"`
+ PublishedAt *time.Time `json:"publishedAt,omitempty"`
+}
diff --git a/pluginsdk/stream.go b/pluginsdk/stream.go
new file mode 100644
index 00000000..9592a9e6
--- /dev/null
+++ b/pluginsdk/stream.go
@@ -0,0 +1,130 @@
+package pluginsdk
+
+import (
+ "encoding/json"
+ "net/http"
+ "sync"
+
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// Stream writes a streaming answer: items, checkpoints and progress as NDJSON
+// lines, flushed one by one. It is safe for concurrent use.
+type Stream struct {
+ mu sync.Mutex
+ w http.ResponseWriter
+ flusher http.Flusher
+ started bool
+ closed bool
+ err error
+}
+
+func newStream(w http.ResponseWriter) *Stream {
+ f, _ := w.(http.Flusher)
+ return &Stream{w: w, flusher: f}
+}
+
+func (s *Stream) write(ev pluginapi.Event) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.err != nil {
+ return s.err
+ }
+ if s.closed {
+ return pluginapi.Errorf(pluginapi.CodeInternal, "stream already ended")
+ }
+ if !s.started {
+ s.w.Header().Set("Content-Type", pluginapi.NDJSONContentType)
+ s.w.WriteHeader(http.StatusOK)
+ s.started = true
+ }
+ b, err := json.Marshal(ev)
+ if err != nil {
+ return err
+ }
+ if _, err := s.w.Write(append(b, '\n')); err != nil {
+ // The caller went away; stop producing.
+ s.err = err
+ return err
+ }
+ if s.flusher != nil {
+ s.flusher.Flush()
+ }
+ return nil
+}
+
+func dataEvent(t pluginapi.EventType, v any) (pluginapi.Event, error) {
+ raw, err := json.Marshal(v)
+ if err != nil {
+ return pluginapi.Event{}, err
+ }
+ return pluginapi.Event{Type: t, Data: raw}, nil
+}
+
+// Item emits one fetched item. An error means WeKnora stopped listening:
+// stop fetching and return.
+func (s *Stream) Item(item pluginapi.FetchedItem) error {
+ ev, err := dataEvent(pluginapi.EventItem, item)
+ if err != nil {
+ return err
+ }
+ return s.write(ev)
+}
+
+// Checkpoint emits a resumable cursor. It must be a complete snapshot.
+func (s *Stream) Checkpoint(c pluginapi.Cursor) error {
+ ev, err := dataEvent(pluginapi.EventCheckpoint, c)
+ if err != nil {
+ return err
+ }
+ return s.write(ev)
+}
+
+// Progress reports progress for display.
+func (s *Stream) Progress(message string) error {
+ return s.write(pluginapi.Event{Type: pluginapi.EventProgress, Message: message})
+}
+
+// Log forwards a line to WeKnora's logs (level: debug, info, warn, error).
+func (s *Stream) Log(level, message string) error {
+ return s.write(pluginapi.Event{Type: pluginapi.EventLog, Level: level, Message: message})
+}
+
+func (s *Stream) end(cursor *pluginapi.Cursor) {
+ if cursor == nil {
+ cursor = &pluginapi.Cursor{}
+ }
+ ev, err := dataEvent(pluginapi.EventEnd, cursor)
+ if err == nil {
+ err = s.write(ev)
+ }
+ if err != nil {
+ s.fail(err)
+ return
+ }
+ s.mu.Lock()
+ s.closed = true
+ s.mu.Unlock()
+}
+
+// fail ends the stream with an error: as an error event once lines went
+// out, or as an ordinary error answer before that.
+func (s *Stream) fail(err error) {
+ s.mu.Lock()
+ started, closed := s.started, s.closed
+ s.mu.Unlock()
+ if closed {
+ return
+ }
+ if !started {
+ writeError(s.w, err)
+ s.mu.Lock()
+ s.closed = true
+ s.mu.Unlock()
+ return
+ }
+ _ = s.write(pluginapi.Event{Type: pluginapi.EventError, Error: toError(err)})
+ s.mu.Lock()
+ s.closed = true
+ s.mu.Unlock()
+}
diff --git a/pluginsdk/websearch.go b/pluginsdk/websearch.go
new file mode 100644
index 00000000..a9eb4289
--- /dev/null
+++ b/pluginsdk/websearch.go
@@ -0,0 +1,57 @@
+package pluginsdk
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+
+ "github.com/Tencent/WeKnora/pluginsdk/pluginapi"
+)
+
+// WebSearcher is a web search provider. Its instance configuration is what
+// the contribution's instanceSchema describes.
+type WebSearcher interface {
+ Search(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error)
+}
+
+// WebSearchFunc adapts a function to WebSearcher.
+type WebSearchFunc func(ctx context.Context, call *Call, in pluginapi.SearchInput) (*pluginapi.SearchOutput, error)
+
+// Search implements WebSearcher.
+func (f WebSearchFunc) Search(
+ ctx context.Context,
+ call *Call,
+ in pluginapi.SearchInput,
+) (*pluginapi.SearchOutput, error) {
+ return f(ctx, call, in)
+}
+
+// WebSearch registers web search provider id (contributes.webSearch[].id).
+func (p *Plugin) WebSearch(id string, s WebSearcher) { p.webSearch[id] = s }
+
+func (p *Plugin) routeWebSearch(mux *http.ServeMux) {
+ mux.HandleFunc("POST /v1/websearch/{id}/search", func(w http.ResponseWriter, r *http.Request) {
+ s, ok := p.webSearch[r.PathValue("id")]
+ if !ok {
+ writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no web search provider %q", r.PathValue("id")))
+ return
+ }
+ p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
+ in, err := decodeInput[pluginapi.SearchInput](raw)
+ if err != nil {
+ return nil, err
+ }
+ out, err := s.Search(ctx, call, in)
+ if err != nil {
+ return nil, err
+ }
+ if out == nil {
+ out = &pluginapi.SearchOutput{}
+ }
+ if out.Results == nil {
+ out.Results = []pluginapi.SearchResult{}
+ }
+ return out, nil
+ })(w, r)
+ })
+}