WeKnora/internal/plugin/activate/connectors.go

355 lines
10 KiB
Go

package activate
import (
"context"
"encoding/json"
"fmt"
"sort"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/plugin/configschema"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
)
// Connectors registers code plugins' data source connectors. A plugin
// connector's instanceSchema splits in two: fields in x-group "settings"
// are non-secret options (DataSourceConfig.Settings), everything else is
// credentials (encrypted at rest).
type Connectors struct {
iv *Invoker
registry *datasource.ConnectorRegistry
mu sync.Mutex
registered map[string][]string
}
// NewConnectors creates the connector activator.
func NewConnectors(iv *Invoker, registry *datasource.ConnectorRegistry) *Connectors {
return &Connectors{iv: iv, registry: registry, registered: map[string][]string{}}
}
// Name implements reconcile.Activator.
func (a *Connectors) Name() string { return "connectors" }
// settingsGroup marks instanceSchema fields that are settings, not
// credentials.
const settingsGroup = "settings"
// Activate implements reconcile.Activator: all of a plugin's connectors or
// none.
func (a *Connectors) Activate(_ context.Context, l *reconcile.Loaded) error {
var ids []string
for _, c := range l.Manifest.Contributes[manifest.PointConnectors] {
id := manifest.QualifiedID(l.Manifest.ID, c.ID)
meta, err := connectorMetadata(l, c, id)
if err == nil {
rc := &remoteConnector{iv: a.iv, m: l.Manifest, local: c.ID, typeID: id}
err = a.registry.RegisterPlugin(rc, meta)
}
if err != nil {
for _, done := range ids {
a.registry.Unregister(done)
}
return fmt.Errorf("connector %s: %w", c.ID, err)
}
ids = append(ids, id)
}
a.mu.Lock()
a.registered[l.Manifest.ID] = ids
a.mu.Unlock()
return nil
}
// Deactivate implements reconcile.Activator.
func (a *Connectors) Deactivate(_ context.Context, pluginID string) error {
a.mu.Lock()
ids := a.registered[pluginID]
delete(a.registered, pluginID)
a.mu.Unlock()
for _, id := range ids {
a.registry.Unregister(id)
}
return nil
}
func connectorMetadata(
l *reconcile.Loaded,
c manifest.Contribution,
id string,
) (datasource.ConnectorMetadata, error) {
src, err := instanceSchema(l.Package, c)
if err != nil {
return datasource.ConnectorMetadata{}, err
}
creds, settings := splitConnectorSchema(src)
icon, err := iconDataURI(l.Package, c.Icon)
if err != nil {
return datasource.ConnectorMetadata{}, err
}
caps := c.Capabilities
if caps == nil {
caps = []string{}
}
return datasource.ConnectorMetadata{
Type: id, Name: c.Name.Default, Names: c.Name.Locales,
Description: c.Description.Default, Descriptions: c.Description.Locales,
Icon: icon, Priority: 1000 + c.Order, AuthType: "plugin", Capabilities: caps,
ConfigSchema: creds, SettingsSchema: settings, PluginID: l.Manifest.ID,
}, nil
}
// splitConnectorSchema separates settings fields from credential fields.
func splitConnectorSchema(src *configschema.Schema) (creds, settings *configschema.Schema) {
if src == nil {
return nil, nil
}
required := map[string]bool{}
for _, k := range src.Required {
required[k] = true
}
creds, settings = configschema.Object(), configschema.Object()
keys := make([]string, 0, len(src.Properties))
for k := range src.Properties {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
prop := *src.Properties[k]
if prop.Group == settingsGroup && !prop.Secret {
prop.Group = ""
settings.Set(k, &prop, required[k])
continue
}
creds.Set(k, &prop, required[k])
}
if len(creds.Properties) == 0 {
creds = nil
}
if len(settings.Properties) == 0 {
settings = nil
}
return creds, settings
}
// remoteConnector is a plugin connector. It streams, so the sync service
// interleaves fetch, ingest and checkpoint and resumes after a timeout.
type remoteConnector struct {
iv *Invoker
m *manifest.Manifest
local string
typeID string
}
var _ datasource.FullStreamingConnector = (*remoteConnector)(nil)
func (r *remoteConnector) Type() string { return r.typeID }
func instanceOf(cfg *types.DataSourceConfig) map[string]any {
if cfg == nil {
return map[string]any{}
}
return map[string]any{
"credentials": nonNil(cfg.Credentials),
"settings": nonNil(cfg.Settings),
"resourceIds": cfg.ResourceIDs,
}
}
func nonNil(m map[string]any) map[string]any {
if m == nil {
return map[string]any{}
}
return m
}
// Validate implements datasource.Connector.
func (r *remoteConnector) Validate(ctx context.Context, cfg *types.DataSourceConfig) error {
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
defer cancel()
return r.iv.Call(ctx, r.m, pluginapi.ConnectorValidatePath(r.local), instanceOf(cfg), nil, nil)
}
// ListResources implements datasource.Connector.
func (r *remoteConnector) ListResources(
ctx context.Context, cfg *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
defer cancel()
var out pluginapi.ListResourcesOutput
err := r.iv.Call(ctx, r.m, pluginapi.ConnectorListResourcesPath(r.local), instanceOf(cfg),
pluginapi.ListResourcesInput{ParentID: parentID}, &out)
if err != nil {
return nil, err
}
res := make([]types.Resource, 0, len(out.Resources))
for _, x := range out.Resources {
item := types.Resource{
ExternalID: x.ExternalID, Name: x.Name, Type: x.Type, Description: x.Description, URL: x.URL,
ParentID: x.ParentID, HasChildren: x.HasChildren, Metadata: x.Metadata,
}
if x.ModifiedAt != nil {
item.ModifiedAt = *x.ModifiedAt
}
res = append(res, item)
}
return res, nil
}
// ResolveResourceAncestors implements datasource.Connector.
func (r *remoteConnector) ResolveResourceAncestors(
ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
) ([]string, error) {
ctx, cancel := withDefaultTimeout(ctx, metadataTimeout)
defer cancel()
var out pluginapi.ResolveAncestorsOutput
err := r.iv.Call(ctx, r.m, pluginapi.ConnectorResolveAncestorsPath(r.local), instanceOf(cfg),
pluginapi.ResolveAncestorsInput{ResourceIDs: resourceIDs}, &out)
return out.Ancestors, err
}
// collector gathers a stream for the non-streaming entry points.
type collector struct {
items []types.FetchedItem
cursor *types.SyncCursor
}
func (c *collector) Emit(_ context.Context, item types.FetchedItem) error {
c.items = append(c.items, item)
return nil
}
func (c *collector) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
c.cursor = cursor
return nil
}
// FetchAll implements datasource.Connector.
func (r *remoteConnector) FetchAll(
ctx context.Context, cfg *types.DataSourceConfig, resourceIDs []string,
) ([]types.FetchedItem, error) {
c := &collector{}
scoped := *cfg
if len(resourceIDs) > 0 {
scoped.ResourceIDs = resourceIDs
}
_, err := r.fetch(ctx, &scoped, pluginapi.FetchFull, nil, c)
return c.items, err
}
// FetchIncremental implements datasource.Connector.
func (r *remoteConnector) FetchIncremental(
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor,
) ([]types.FetchedItem, *types.SyncCursor, error) {
c := &collector{}
next, err := r.fetch(ctx, cfg, pluginapi.FetchIncremental, cursor, c)
return c.items, next, err
}
// FetchStream implements datasource.StreamingConnector: incremental from a
// cursor, full without one.
func (r *remoteConnector) FetchStream(
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
) (*types.SyncCursor, error) {
mode := pluginapi.FetchIncremental
if cursor == nil {
mode = pluginapi.FetchFull
}
return r.fetch(ctx, cfg, mode, cursor, h)
}
// FetchFullStream implements datasource.FullStreamingConnector: every item
// again, with the previous cursor for deletion reconciliation.
func (r *remoteConnector) FetchFullStream(
ctx context.Context, cfg *types.DataSourceConfig, cursor *types.SyncCursor, h datasource.StreamHandler,
) (*types.SyncCursor, error) {
return r.fetch(ctx, cfg, pluginapi.FetchFull, cursor, h)
}
func (r *remoteConnector) fetch(
ctx context.Context, cfg *types.DataSourceConfig, mode string, cursor *types.SyncCursor,
h datasource.StreamHandler,
) (*types.SyncCursor, error) {
in := pluginapi.FetchInput{Mode: mode, Cursor: toPluginCursor(cursor)}
if cfg != nil {
in.ResourceIDs = cfg.ResourceIDs
}
end, err := r.iv.Stream(ctx, r.m, pluginapi.ConnectorFetchPath(r.local), instanceOf(cfg), in,
func(ev pluginapi.Event) error {
switch ev.Type {
case pluginapi.EventItem:
var it pluginapi.FetchedItem
if err := json.Unmarshal(ev.Data, &it); err != nil {
return fmt.Errorf("decode fetched item: %w", err)
}
return h.Emit(ctx, fromPluginItem(it))
case pluginapi.EventCheckpoint:
var c pluginapi.Cursor
if err := json.Unmarshal(ev.Data, &c); err != nil {
return fmt.Errorf("decode checkpoint: %w", err)
}
return h.Checkpoint(ctx, fromPluginCursor(&c))
}
return nil
})
if err != nil {
return nil, err
}
var final pluginapi.Cursor
if len(end) > 0 {
if err := json.Unmarshal(end, &final); err != nil {
return nil, fmt.Errorf("decode final cursor: %w", err)
}
}
next := fromPluginCursor(&final)
if next.LastSyncTime.IsZero() {
next.LastSyncTime = time.Now()
}
return next, nil
}
func toPluginCursor(c *types.SyncCursor) *pluginapi.Cursor {
if c == nil {
return nil
}
out := &pluginapi.Cursor{State: c.ConnectorCursor}
if !c.LastSyncTime.IsZero() {
t := c.LastSyncTime
out.LastSyncTime = &t
}
return out
}
func fromPluginCursor(c *pluginapi.Cursor) *types.SyncCursor {
out := &types.SyncCursor{ConnectorCursor: c.State}
if c.LastSyncTime != nil {
out.LastSyncTime = *c.LastSyncTime
}
if out.ConnectorCursor == nil {
out.ConnectorCursor = map[string]any{}
}
return out
}
func fromPluginItem(it pluginapi.FetchedItem) types.FetchedItem {
out := types.FetchedItem{
ExternalID: it.ExternalID, Title: it.Title, Content: it.Content, ContentType: it.ContentType,
FileName: it.FileName, URL: it.URL, Metadata: it.Metadata, IsDeleted: it.IsDeleted,
SourceResourceID: it.SourceResourceID,
}
if it.UpdatedAt != nil {
out.UpdatedAt = *it.UpdatedAt
}
if it.CreatedAt != nil {
out.CreatedAt = *it.CreatedAt
}
if out.ContentType == "" {
out.ContentType = "text/markdown"
}
return out
}