WeKnora/internal/plugin/reconcile/reconcile.go

512 lines
14 KiB
Go

// Package reconcile makes every node run the plugins the database says should
// run. Installing, upgrading, disabling or removing a plugin only writes rows;
// each node's Reconciler then loads the active version of every enabled plugin
// into the registry, hands its contributions to the domain activators, and
// unloads everything else. A Redis broadcast makes peers reconcile at once; a
// periodic pass catches anything a broadcast missed.
package reconcile
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/plugin/manifest"
"github.com/Tencent/WeKnora/internal/plugin/pkg"
"github.com/Tencent/WeKnora/internal/plugin/registry"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
)
// Loaded is a plugin version loaded on this node.
type Loaded struct {
Manifest *manifest.Manifest
Package *pkg.Package
// Dir is the package extracted on local disk, for consumers that read
// files by path (skills).
Dir string
// Installed is the plugin's row: where a remote plugin runs and its
// sealed secret.
Installed types.InstalledPlugin
}
// Activator wires one domain to plugin contributions: it registers what a
// loaded plugin contributes (skills, MCP servers, model vendors) and removes
// it again. Activate is called again for each new version, after Deactivate
// for the old one.
type Activator interface {
Name() string
Activate(ctx context.Context, l *Loaded) error
Deactivate(ctx context.Context, pluginID string) error
}
// InPlaceActivator swaps an upgraded plugin itself inside Activate (the host
// starts the new process before stopping the old one), so it is not
// deactivated first.
type InPlaceActivator interface {
Activator
ActivatesInPlace()
}
// Status is how a plugin fares on this node.
type Status struct {
Version string `json:"version"`
State string `json:"state"`
Error string `json:"error,omitempty"`
UpdatedAt time.Time `json:"updatedAt"`
}
// Node states reported in Status.
const (
StateReady = "ready"
StateFailed = "failed"
// StateDegraded: loaded, but its process crashed or fails health checks
// and is being restarted.
StateDegraded = "degraded"
)
// DefaultInterval is how often a node reconciles without being told to.
const DefaultInterval = 30 * time.Second
const channelBase = "weknora:plugins:changed"
func channel() string {
if ns := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_NAMESPACE")); ns != "" {
return channelBase + ":" + ns
}
return channelBase
}
type changeMessage struct {
OriginID string `json:"origin_id"`
}
// Reconciler converges this node to the installed-plugin rows.
type Reconciler struct {
repo interfaces.PluginRepository
store PackageStore
registry *registry.Registry
cacheDir string
rdb *redis.Client
activators []Activator
instanceID string
interval time.Duration
runtimes map[string]bool
accept func(*manifest.Manifest) bool
role string
mu sync.Mutex // serializes passes
loaded map[string]*Loaded
digests map[string]string // plugin ID → loaded digest and runtime target
// retries holds plugins whose activation failed: a process that would
// not start, a remote service that was down. They are tried again with
// backoff until they load or change.
retries map[string]retry
now func() time.Time
statusMu sync.RWMutex
status map[string]Status
runOnce sync.Once
}
// retry is when a failed activation is tried again.
type retry struct {
key string // the load key that failed
attempts int
next time.Time
}
// Backoff between attempts to activate a plugin that failed.
const (
retryFloor = 30 * time.Second
retryCeiling = 10 * time.Minute
)
func retryDelay(attempts int) time.Duration {
d := retryFloor
for i := 1; i < attempts && d < retryCeiling; i++ {
d *= 2
}
return min(d, retryCeiling)
}
// Options configures a Reconciler.
type Options struct {
Repo interfaces.PluginRepository
Store PackageStore
Registry *registry.Registry
CacheDir string
Redis *redis.Client // nil on single-node deployments
Activators []Activator
Interval time.Duration
// Runtimes limits the node to plugins of these runtimes (a standalone
// plugin host loads host plugins only); empty means all.
Runtimes []manifest.RuntimeType
// Accept further limits the node by the active version's manifest (a
// plugin host that runs python plugins only); nil accepts all.
Accept func(*manifest.Manifest) bool
// Role names what the node is in status reports, e.g. "plugin-host".
Role string
}
// New creates a Reconciler.
func New(o Options) *Reconciler {
if o.Interval <= 0 {
o.Interval = DefaultInterval
}
if o.CacheDir == "" {
o.CacheDir = DefaultCacheDir()
}
var runtimes map[string]bool
if len(o.Runtimes) > 0 {
runtimes = map[string]bool{}
for _, rt := range o.Runtimes {
runtimes[string(rt)] = true
}
}
return &Reconciler{
repo: o.Repo, store: o.Store, registry: o.Registry, cacheDir: o.CacheDir, rdb: o.Redis,
activators: o.Activators, instanceID: uuid.NewString(), interval: o.Interval,
runtimes: runtimes, accept: o.Accept, role: o.Role,
loaded: map[string]*Loaded{}, digests: map[string]string{}, retries: map[string]retry{},
status: map[string]Status{}, now: time.Now,
}
}
// DefaultCacheDir is where packages are extracted unless
// WEKNORA_PLUGIN_CACHE_DIR says otherwise.
func DefaultCacheDir() string {
if dir := strings.TrimSpace(os.Getenv("WEKNORA_PLUGIN_CACHE_DIR")); dir != "" {
return dir
}
return filepath.Join(os.TempDir(), "weknora-plugins")
}
// Reconcile runs one pass. It keeps going past a plugin that fails to load,
// records the failure in Status, and returns every failure joined.
func (r *Reconciler) Reconcile(ctx context.Context) error {
r.mu.Lock()
defer r.mu.Unlock()
rows, err := r.repo.ListPlugins(ctx)
if err != nil {
return fmt.Errorf("list installed plugins: %w", err)
}
var errs []error
desired := map[string]bool{}
for _, row := range rows {
if row.DesiredState != types.PluginStateEnabled || (r.runtimes != nil && !r.runtimes[row.Runtime]) ||
!r.accepts(ctx, row) {
continue
}
desired[row.ID] = true
if err := r.ensure(ctx, row); err != nil {
errs = append(errs, fmt.Errorf("plugin %s: %w", row.ID, err))
r.setStatus(row.ID, Status{Version: row.ActiveVersion, State: StateFailed, Error: err.Error()})
}
}
for id := range r.loaded {
if !desired[id] {
r.unload(ctx, id)
}
}
// A plugin that never loaded has a status but no digest.
r.statusMu.Lock()
for id := range r.status {
if !desired[id] {
delete(r.status, id)
r.forgetStatus(ctx, id)
}
}
r.statusMu.Unlock()
r.publishStatuses(ctx)
return errors.Join(errs...)
}
// accepts applies Options.Accept to a plugin's active version. A version
// that cannot be read is accepted, so ensure reports why.
func (r *Reconciler) accepts(ctx context.Context, row types.InstalledPlugin) bool {
if r.accept == nil {
return true
}
v, err := r.repo.GetVersion(ctx, row.ID, row.ActiveVersion)
if err != nil || v == nil {
return true
}
var m manifest.Manifest
if json.Unmarshal(v.Manifest, &m) != nil {
return true
}
return r.accept(&m)
}
// runtimeTarget identifies where a plugin runs beyond its package: a new
// remote URL or secret reloads the plugin like a new version would.
func runtimeTarget(row types.InstalledPlugin) string {
if row.RemoteURL == "" && row.RemoteSecret == "" {
return ""
}
sum := sha256.Sum256([]byte(row.RemoteURL + "\x00" + row.RemoteSecret))
return hex.EncodeToString(sum[:8])
}
// ensure loads the active version of one plugin unless it already is.
func (r *Reconciler) ensure(ctx context.Context, row types.InstalledPlugin) error {
v, err := r.repo.GetVersion(ctx, row.ID, row.ActiveVersion)
if err != nil {
return err
}
if v == nil {
return fmt.Errorf("version %s is not stored", row.ActiveVersion)
}
loadKey := v.Digest + "|" + runtimeTarget(row)
if r.digests[row.ID] == loadKey {
return nil
}
if rt, ok := r.retries[row.ID]; ok && rt.key == loadKey && r.now().Before(rt.next) {
return nil // still failed; its status says why
}
data, err := r.store.Get(ctx, v.PackageURI)
if err != nil {
return err
}
p, err := pkg.Open(data)
if err != nil {
return err
}
if p.Digest != v.Digest {
return fmt.Errorf("stored package digest %s does not match %s", p.Digest, v.Digest)
}
if p.Manifest.ID != row.ID {
return fmt.Errorf("package is plugin %s, not %s", p.Manifest.ID, row.ID)
}
dir, err := r.extract(p)
if err != nil {
return err
}
if err := r.registry.Replace(p.Manifest); err != nil {
return err
}
l := &Loaded{Manifest: p.Manifest, Package: p, Dir: dir, Installed: row}
var errs []error
for _, a := range r.activators {
_, inPlace := a.(InPlaceActivator)
if _, had := r.loaded[row.ID]; had && !inPlace {
if err := a.Deactivate(ctx, row.ID); err != nil {
errs = append(errs, fmt.Errorf("%s: deactivate previous version: %w", a.Name(), err))
}
}
if err := a.Activate(ctx, l); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", a.Name(), err))
}
}
r.loaded[row.ID] = l
if err := errors.Join(errs...); err != nil {
rt := r.retries[row.ID]
if rt.key != loadKey {
rt = retry{key: loadKey}
}
rt.attempts++
rt.next = r.now().Add(retryDelay(rt.attempts))
r.retries[row.ID] = rt
delete(r.digests, row.ID)
return err
}
r.digests[row.ID] = loadKey
delete(r.retries, row.ID)
logger.Infof(ctx, "[plugin] loaded %s %s", row.ID, row.ActiveVersion)
r.setStatus(row.ID, Status{Version: row.ActiveVersion, State: StateReady})
return nil
}
func (r *Reconciler) unload(ctx context.Context, id string) {
// Reverse order: routes go before the processes they route to.
for i := len(r.activators) - 1; i >= 0; i-- {
a := r.activators[i]
if err := a.Deactivate(ctx, id); err != nil {
logger.Warnf(ctx, "[plugin] %s: deactivate %s: %v", a.Name(), id, err)
}
}
if err := r.registry.Unregister(id); err != nil {
logger.Warnf(ctx, "[plugin] unregister %s: %v", id, err)
}
delete(r.loaded, id)
delete(r.digests, id)
delete(r.retries, id)
r.statusMu.Lock()
delete(r.status, id)
r.statusMu.Unlock()
r.forgetStatus(ctx, id)
logger.Infof(ctx, "[plugin] unloaded %s", id)
}
// extract writes the package under cacheDir/<digest>, once per digest.
func (r *Reconciler) extract(p *pkg.Package) (string, error) {
dir := filepath.Join(r.cacheDir, strings.TrimPrefix(p.Digest, "sha256:"))
if _, err := os.Stat(filepath.Join(dir, pkg.ManifestFile)); err == nil {
return dir, nil
}
tmp := dir + ".tmp-" + uuid.NewString()[:8]
for _, name := range p.Files("") {
data, _ := p.ReadFile(name)
target, err := utils.SafeJoinUnderBase(tmp, name)
if err != nil {
_ = os.RemoveAll(tmp)
return "", err
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
_ = os.RemoveAll(tmp)
return "", err
}
if err := os.WriteFile(target, data, 0o644); err != nil {
_ = os.RemoveAll(tmp)
return "", err
}
}
if err := os.Rename(tmp, dir); err != nil {
_ = os.RemoveAll(tmp)
// Another pass extracted the same digest first.
if _, statErr := os.Stat(filepath.Join(dir, pkg.ManifestFile)); statErr == nil {
return dir, nil
}
return "", err
}
return dir, nil
}
func (r *Reconciler) setStatus(id string, s Status) {
s.UpdatedAt = time.Now()
r.statusMu.Lock()
r.status[id] = s
r.statusMu.Unlock()
}
// ReportRuntime records a loaded plugin's runtime health (a host process that
// crashed and is restarting, or recovered) and publishes it at once.
func (r *Reconciler) ReportRuntime(pluginID string, healthy bool, err error) {
r.statusMu.Lock()
s, ok := r.status[pluginID]
if !ok || s.State == StateFailed {
r.statusMu.Unlock()
return
}
s.State, s.Error = StateReady, ""
if !healthy {
s.State = StateDegraded
if err != nil {
s.Error = err.Error()
}
}
s.UpdatedAt = time.Now()
r.status[pluginID] = s
r.statusMu.Unlock()
r.publishStatuses(context.Background())
}
// Status reports how one plugin fares on this node.
func (r *Reconciler) Status(pluginID string) (Status, bool) {
r.statusMu.RLock()
defer r.statusMu.RUnlock()
s, ok := r.status[pluginID]
return s, ok
}
// Loaded returns the plugins loaded on this node, sorted by ID.
func (r *Reconciler) Loaded() []*Loaded {
r.mu.Lock()
defer r.mu.Unlock()
out := make([]*Loaded, 0, len(r.loaded))
for _, l := range r.loaded {
out = append(out, l)
}
sort.Slice(out, func(i, j int) bool { return out[i].Manifest.ID < out[j].Manifest.ID })
return out
}
// Notify tells peer nodes to reconcile. Best effort: the periodic pass
// catches up when Redis is down or absent.
func (r *Reconciler) Notify(ctx context.Context) {
if r.rdb == nil {
return
}
payload, _ := json.Marshal(changeMessage{OriginID: r.instanceID})
if err := r.rdb.Publish(ctx, channel(), payload).Err(); err != nil {
logger.Warnf(ctx, "[plugin] publish change: %v", err)
}
}
// Start reconciles once, then keeps reconciling on broadcasts and on a timer
// until ctx ends. Calling it twice has no effect.
func (r *Reconciler) Start(ctx context.Context) {
r.runOnce.Do(func() {
if err := r.Reconcile(ctx); err != nil {
logger.Warnf(ctx, "[plugin] initial reconcile: %v", err)
}
wake := make(chan struct{}, 1)
if r.rdb != nil {
go r.subscribe(ctx, wake)
}
go r.loop(ctx, wake)
})
}
func (r *Reconciler) loop(ctx context.Context, wake <-chan struct{}) {
t := time.NewTicker(r.interval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
case <-wake:
}
if err := r.Reconcile(ctx); err != nil {
logger.Warnf(ctx, "[plugin] reconcile: %v", err)
}
}
}
// subscribe forwards peer broadcasts to wake, reconnecting with backoff.
func (r *Reconciler) subscribe(ctx context.Context, wake chan<- struct{}) {
const maxBackoff = 30 * time.Second
backoff := time.Second
for ctx.Err() == nil {
sub := r.rdb.Subscribe(ctx, channel())
if _, err := sub.Receive(ctx); err != nil {
_ = sub.Close()
logger.Warnf(ctx, "[plugin] subscribe: %v (retry in %s)", err, backoff)
select {
case <-time.After(backoff):
case <-ctx.Done():
return
}
backoff = min(backoff*2, maxBackoff)
continue
}
backoff = time.Second
for msg := range sub.Channel() {
var m changeMessage
if json.Unmarshal([]byte(msg.Payload), &m) == nil && m.OriginID == r.instanceID {
continue
}
select {
case wake <- struct{}{}:
default: // a pass is already pending
}
}
_ = sub.Close()
}
}