155 lines
4.5 KiB
Go
155 lines
4.5 KiB
Go
package reconcile
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/plugin/driver"
|
|
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
|
)
|
|
|
|
// NodeStatus is one node's report on a plugin.
|
|
type NodeStatus struct {
|
|
Node string `json:"node"`
|
|
Status
|
|
// SeenAt is when the node last confirmed the report.
|
|
SeenAt time.Time `json:"seenAt"`
|
|
}
|
|
|
|
const statusKeyBase = "weknora:plugins:status"
|
|
|
|
func statusKey(pluginID string) string {
|
|
if ns := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_NAMESPACE")); ns != "" {
|
|
return statusKeyBase + ":" + ns + ":" + pluginID
|
|
}
|
|
return statusKeyBase + ":" + pluginID
|
|
}
|
|
|
|
// NodeName identifies this node in status reports.
|
|
func (r *Reconciler) NodeName() string {
|
|
host, _ := os.Hostname()
|
|
if host == "" {
|
|
host = "node"
|
|
}
|
|
if r.role != "" {
|
|
host = r.role + ":" + host
|
|
}
|
|
return host + "/" + r.instanceID[:8]
|
|
}
|
|
|
|
// publishStatuses reports every plugin this node knows about to Redis, so
|
|
// any node can show the whole cluster. Reports refresh on every pass; a node
|
|
// that stops reporting ages out.
|
|
func (r *Reconciler) publishStatuses(ctx context.Context) {
|
|
if r.rdb == nil {
|
|
return
|
|
}
|
|
r.statusMu.RLock()
|
|
snapshot := make(map[string]Status, len(r.status))
|
|
for id, s := range r.status {
|
|
snapshot[id] = s
|
|
}
|
|
r.statusMu.RUnlock()
|
|
node, now := r.NodeName(), time.Now()
|
|
pipe := r.rdb.Pipeline()
|
|
for id, s := range snapshot {
|
|
b, _ := json.Marshal(NodeStatus{Node: node, Status: s, SeenAt: now})
|
|
pipe.HSet(ctx, statusKey(id), node, b)
|
|
pipe.Expire(ctx, statusKey(id), r.staleAfter()*2)
|
|
}
|
|
if _, err := pipe.Exec(ctx); err != nil {
|
|
logger.Warnf(ctx, "[plugin] publish node status: %v", err)
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) forgetStatus(ctx context.Context, pluginID string) {
|
|
if r.rdb == nil {
|
|
return
|
|
}
|
|
if err := r.rdb.HDel(ctx, statusKey(pluginID), r.NodeName()).Err(); err != nil {
|
|
logger.Warnf(ctx, "[plugin] clear node status of %s: %v", pluginID, err)
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) staleAfter() time.Duration { return 3 * r.interval }
|
|
|
|
// NodeStatuses reports a plugin on every live node; without Redis, on this
|
|
// node only.
|
|
func (r *Reconciler) NodeStatuses(ctx context.Context, pluginID string) ([]NodeStatus, error) {
|
|
local, hasLocal := r.Status(pluginID)
|
|
if r.rdb == nil {
|
|
if !hasLocal {
|
|
return nil, nil
|
|
}
|
|
return []NodeStatus{{Node: r.NodeName(), Status: local, SeenAt: time.Now()}}, nil
|
|
}
|
|
fields, err := r.rdb.HGetAll(ctx, statusKey(pluginID)).Result()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read node status: %w", err)
|
|
}
|
|
var out []NodeStatus
|
|
self := r.NodeName()
|
|
for node, raw := range fields {
|
|
var ns NodeStatus
|
|
if json.Unmarshal([]byte(raw), &ns) != nil || node == self || time.Since(ns.SeenAt) > r.staleAfter() {
|
|
continue
|
|
}
|
|
out = append(out, ns)
|
|
}
|
|
if hasLocal {
|
|
out = append(out, NodeStatus{Node: self, Status: local, SeenAt: time.Now()})
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
|
return out, nil
|
|
}
|
|
|
|
// Driver returns the status driver for plugins the reconciler loads on every
|
|
// node (declarative and host): an instance is a node that loaded the plugin,
|
|
// with the runtime health the node reported.
|
|
func (r *Reconciler) Driver(rt manifest.RuntimeType) driver.Driver { return nodeDriver{r: r, rt: rt} }
|
|
|
|
type nodeDriver struct {
|
|
r *Reconciler
|
|
rt manifest.RuntimeType
|
|
}
|
|
|
|
func (d nodeDriver) Type() manifest.RuntimeType { return d.rt }
|
|
|
|
// Ensure and Remove are the reconciler's job.
|
|
func (nodeDriver) Ensure(context.Context, *manifest.Manifest) error { return nil }
|
|
func (nodeDriver) Remove(context.Context, string, string) error { return nil }
|
|
|
|
// Resolve is not how these plugins are reached: declarative ones have no
|
|
// endpoint and host ones are reached through the node's host manager.
|
|
func (d nodeDriver) Resolve(_ context.Context, pluginID string, _ uint64) (driver.Endpoint, error) {
|
|
return driver.Endpoint{}, fmt.Errorf("%s plugin %s has no shared endpoint", d.rt, pluginID)
|
|
}
|
|
|
|
func (d nodeDriver) Status(ctx context.Context, pluginID string) ([]driver.InstanceStatus, error) {
|
|
nodes, err := d.r.NodeStatuses(ctx, pluginID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]driver.InstanceStatus, 0, len(nodes))
|
|
for _, n := range nodes {
|
|
state := driver.StateReady
|
|
switch n.State {
|
|
case StateReady:
|
|
case StateDegraded:
|
|
state = driver.StateDegraded
|
|
default:
|
|
state = driver.StateStopped
|
|
}
|
|
out = append(out, driver.InstanceStatus{
|
|
Node: n.Node, Version: n.Version, State: state, Error: n.Error, UpdatedAt: n.UpdatedAt,
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|