fix(im): harden connection lifecycle across replicas
parent
4826de44de
commit
53ddad57c0
|
|
@ -378,7 +378,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
// IM integration
|
||||
logger.Debugf(ctx, "[Container] Registering IM integration...")
|
||||
must(container.Provide(imPkg.NewService))
|
||||
must(container.Invoke(registerIMAdapterFactories))
|
||||
must(container.Invoke(registerIMService))
|
||||
must(container.Provide(handler.NewIMHandler))
|
||||
must(container.Provide(handler.NewEmbedChannelHandler))
|
||||
must(container.Provide(handler.NewWeKnoraCloudHandler))
|
||||
|
|
@ -1542,10 +1542,10 @@ func registerWebSearchProviders(registry *infra_web_search.Registry) {
|
|||
registry.Register("zhipu", infra_web_search.NewZhipuProvider)
|
||||
}
|
||||
|
||||
// registerIMAdapterFactories registers adapter factories for each IM platform
|
||||
// and loads enabled channels from the database. Each platform's factory lives
|
||||
// in its own subpackage to keep this file focused on wiring.
|
||||
func registerIMAdapterFactories(imService *imPkg.Service) {
|
||||
// registerIMService registers adapter factories, loads enabled channels, and
|
||||
// wires the process-lifetime shutdown hook. Each platform's factory lives in
|
||||
// its own subpackage to keep this file focused on wiring.
|
||||
func registerIMService(imService *imPkg.Service, cleaner interfaces.ResourceCleaner) {
|
||||
imService.RegisterAdapterFactory("wecom", wecom.NewFactory())
|
||||
imService.RegisterAdapterFactory("feishu", feishu.NewFactory(feishu.RegionFeishu))
|
||||
// Lark is Feishu's international cloud: same adapter, different host/tenant.
|
||||
|
|
@ -1562,6 +1562,11 @@ func registerIMAdapterFactories(imService *imPkg.Service) {
|
|||
if err := imService.LoadAndStartChannels(); err != nil {
|
||||
logger.Warnf(context.Background(), "[IM] Failed to load channels from database: %v", err)
|
||||
}
|
||||
|
||||
cleaner.RegisterWithName("IMService", func() error {
|
||||
imService.Stop()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// initConnectorRegistry creates and populates the connector registry with all available connectors.
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
|
@ -10,6 +11,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// validIMPlatforms is the set of supported IM platforms.
|
||||
|
|
@ -364,29 +366,18 @@ func (h *IMHandler) IMCallback(c *gin.Context) {
|
|||
ctx := c.Request.Context()
|
||||
channelID := c.Param("channel_id")
|
||||
|
||||
adapter, channel, ok := h.imService.GetChannelAdapter(channelID)
|
||||
if !ok {
|
||||
// Try loading from DB
|
||||
ch, err := h.imService.GetChannelByID(channelID)
|
||||
if err != nil {
|
||||
// Always validate the durable row before using a cached webhook adapter.
|
||||
// This is the correctness fallback when a replica missed Redis invalidation
|
||||
// while disconnected, and prevents stale credentials/config from being used.
|
||||
adapter, channel, err := h.imService.EnsureChannelAdapter(channelID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
logger.Errorf(ctx, "[IM] Channel not found for callback: %s", channelID)
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "channel not found"})
|
||||
return
|
||||
}
|
||||
if err := h.imService.StartChannel(ch); err != nil {
|
||||
logger.Errorf(ctx, "[IM] Failed to start channel for callback: %v", err)
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"})
|
||||
return
|
||||
}
|
||||
adapter, channel, ok = h.imService.GetChannelAdapter(channelID)
|
||||
if !ok {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if !channel.Enabled {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel is disabled"})
|
||||
logger.Errorf(ctx, "[IM] Channel unavailable for callback %s: %v", channelID, err)
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"})
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,15 +3,18 @@ package im
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
|
||||
|
|
@ -342,6 +345,9 @@ const (
|
|||
RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter
|
||||
RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting
|
||||
RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter
|
||||
// RedisChannelConfig broadcasts durable im_channels mutations so every
|
||||
// application replica invalidates its local adapter/config snapshot.
|
||||
RedisChannelConfig = "im:channel:config"
|
||||
|
||||
defaultRateLimitWindow = 60 * time.Second
|
||||
defaultRateLimitMaxRequests = 10
|
||||
|
|
@ -355,6 +361,15 @@ type channelState struct {
|
|||
leaderCancel context.CancelFunc // stops the leader renewal goroutine (nil if not leader)
|
||||
}
|
||||
|
||||
type leaderRetryState struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type channelConfigEvent struct {
|
||||
ChannelID string `json:"channel_id"`
|
||||
SourceInstance string `json:"source_instance"`
|
||||
}
|
||||
|
||||
// AdapterFactory creates an Adapter from an IMChannel configuration.
|
||||
// The second return value is an optional cleanup function (e.g., for stopping websocket connections).
|
||||
type AdapterFactory func(ctx context.Context, channel *IMChannel, msgHandler func(ctx context.Context, msg *IncomingMessage) error) (Adapter, context.CancelFunc, error)
|
||||
|
|
@ -407,8 +422,9 @@ type Service struct {
|
|||
cmdRegistry *CommandRegistry
|
||||
|
||||
// channels maps channel ID -> running channel state
|
||||
channels map[string]*channelState
|
||||
mu sync.RWMutex
|
||||
channels map[string]*channelState
|
||||
leaderRetries map[string]*leaderRetryState
|
||||
mu sync.RWMutex
|
||||
|
||||
// adapterFactories maps platform name -> factory function
|
||||
adapterFactories map[string]AdapterFactory
|
||||
|
|
@ -438,7 +454,10 @@ type Service struct {
|
|||
// instanceID uniquely identifies this service instance for leader election.
|
||||
instanceID string
|
||||
|
||||
stopCh chan struct{}
|
||||
stopCh chan struct{}
|
||||
stopOnce sync.Once
|
||||
subscriberOnce sync.Once
|
||||
stopped atomic.Bool
|
||||
}
|
||||
|
||||
// makeUserKey builds the canonical key used to identify a user's request
|
||||
|
|
@ -843,6 +862,7 @@ func NewService(
|
|||
oauthManager: oauthManager,
|
||||
cmdRegistry: registry,
|
||||
channels: make(map[string]*channelState),
|
||||
leaderRetries: make(map[string]*leaderRetryState),
|
||||
adapterFactories: make(map[string]AdapterFactory),
|
||||
rateLimiter: ratelimit.New(redisClient, RedisKeyRateLimit, rlWindow, instanceID),
|
||||
rateLimitMax: rlMax,
|
||||
|
|
@ -887,13 +907,22 @@ func (s *Service) RegisterAdapterFactory(platform string, factory AdapterFactory
|
|||
|
||||
// Stop gracefully shuts down the service, stopping all channels and background goroutines.
|
||||
func (s *Service) Stop() {
|
||||
close(s.stopCh)
|
||||
s.qaQueue.Stop()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id, cs := range s.channels {
|
||||
s.stopChannelLocked(id, cs)
|
||||
}
|
||||
s.stopOnce.Do(func() {
|
||||
s.stopped.Store(true)
|
||||
close(s.stopCh)
|
||||
if s.qaQueue != nil {
|
||||
s.qaQueue.Stop()
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id, cs := range s.channels {
|
||||
s.stopChannelLocked(id, cs)
|
||||
}
|
||||
for id, retry := range s.leaderRetries {
|
||||
retry.cancel()
|
||||
delete(s.leaderRetries, id)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// dedupCleanupLoop periodically cleans up expired entries from the dedup map.
|
||||
|
|
@ -918,6 +947,8 @@ func (s *Service) dedupCleanupLoop() {
|
|||
|
||||
// LoadAndStartChannels loads all enabled channels from the database and starts them.
|
||||
func (s *Service) LoadAndStartChannels() error {
|
||||
s.startChannelConfigSubscriber()
|
||||
|
||||
ctx := context.Background()
|
||||
var channels []IMChannel
|
||||
if err := s.db.Where("enabled = ? AND deleted_at IS NULL", true).Find(&channels).Error; err != nil {
|
||||
|
|
@ -928,8 +959,13 @@ func (s *Service) LoadAndStartChannels() error {
|
|||
ch := channels[i]
|
||||
if err := s.StartChannel(&ch); err != nil {
|
||||
logger.Warnf(ctx, "[IM] Failed to start channel %s (%s/%s): %v", ch.ID, ch.Platform, ch.Name, err)
|
||||
} else if _, _, active := s.GetChannelAdapter(ch.ID); active {
|
||||
// Adapter initialization can launch an asynchronous connection attempt.
|
||||
// Do not claim network readiness here; platform connection logs report it.
|
||||
logger.Infof(ctx, "[IM] Initialized channel runtime: id=%s platform=%s name=%s mode=%s agent=%s",
|
||||
ch.ID, ch.Platform, ch.Name, ch.Mode, ch.AgentID)
|
||||
} else {
|
||||
logger.Infof(ctx, "[IM] Started channel: id=%s platform=%s name=%s mode=%s agent=%s",
|
||||
logger.Infof(ctx, "[IM] Channel runtime is on standby: id=%s platform=%s name=%s mode=%s agent=%s",
|
||||
ch.ID, ch.Platform, ch.Name, ch.Mode, ch.AgentID)
|
||||
}
|
||||
}
|
||||
|
|
@ -938,12 +974,98 @@ func (s *Service) LoadAndStartChannels() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// startChannelConfigSubscriber listens for durable channel mutations made by
|
||||
// other application replicas. Redis Pub/Sub provides the fast path; callback
|
||||
// freshness checks and the leader renewal DB check remain the durable fallback
|
||||
// if an event is missed while a replica is disconnected.
|
||||
func (s *Service) startChannelConfigSubscriber() {
|
||||
if s.redis == nil {
|
||||
return
|
||||
}
|
||||
s.subscriberOnce.Do(func() {
|
||||
go s.channelConfigSubscriberLoop()
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) channelConfigSubscriberLoop() {
|
||||
pubsub := s.redis.Subscribe(context.Background(), RedisChannelConfig)
|
||||
defer pubsub.Close()
|
||||
|
||||
messages := pubsub.Channel()
|
||||
for {
|
||||
select {
|
||||
case <-s.stopCh:
|
||||
return
|
||||
case msg, ok := <-messages:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var event channelConfigEvent
|
||||
if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil {
|
||||
logger.Warnf(context.Background(), "[IM] Ignore invalid channel config event: %v", err)
|
||||
continue
|
||||
}
|
||||
if event.ChannelID == "" || event.SourceInstance == s.instanceID {
|
||||
continue
|
||||
}
|
||||
s.reloadChannelFromDB(event.ChannelID, "config event")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) publishChannelConfigChange(channelID string) {
|
||||
if s.redis == nil || channelID == "" || s.stopped.Load() {
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(channelConfigEvent{
|
||||
ChannelID: channelID,
|
||||
SourceInstance: s.instanceID,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := s.redis.Publish(context.Background(), RedisChannelConfig, payload).Err(); err != nil {
|
||||
// The database is the source of truth. Subscribers also verify freshness
|
||||
// on webhook callbacks / leader renewal, so publication is best-effort.
|
||||
logger.Warnf(context.Background(), "[IM] Publish channel config event failed for %s: %v", channelID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) reloadChannelFromDB(channelID, reason string) {
|
||||
fresh, err := s.GetChannelByID(channelID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
s.StopChannel(channelID)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(context.Background(), "[IM] Reload channel %s after %s failed: %v", channelID, reason, err)
|
||||
return
|
||||
}
|
||||
if !fresh.Enabled {
|
||||
s.StopChannel(channelID)
|
||||
return
|
||||
}
|
||||
if _, cached, running := s.GetChannelAdapter(channelID); running && sameChannelRuntimeConfig(cached, fresh) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(context.Background(), "[IM] Reloading channel %s after %s", channelID, reason)
|
||||
if err := s.StartChannel(fresh); err != nil && !s.stopped.Load() {
|
||||
logger.Warnf(context.Background(), "[IM] Reload channel %s after %s failed: %v", channelID, reason, err)
|
||||
}
|
||||
}
|
||||
|
||||
// StartChannel creates and registers an adapter for the given channel.
|
||||
// For WebSocket channels with Redis available, only one instance acquires
|
||||
// the leader lock and opens the connection; other instances periodically
|
||||
// retry so they can take over if the leader dies.
|
||||
func (s *Service) StartChannel(channel *IMChannel) error {
|
||||
if s.stopped.Load() {
|
||||
return fmt.Errorf("im service is stopped")
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.stopLeaderRetryLocked(channel.ID)
|
||||
factory, ok := s.adapterFactories[channel.Platform]
|
||||
if !ok {
|
||||
s.mu.Unlock()
|
||||
|
|
@ -963,7 +1085,7 @@ func (s *Service) StartChannel(channel *IMChannel) error {
|
|||
if !acquired {
|
||||
logger.Infof(context.Background(),
|
||||
"[IM] Channel %s %s owned by another instance, will retry", channel.ID, channel.Mode)
|
||||
go s.wsLeaderRetryLoop(channel)
|
||||
s.scheduleWSLeaderRetry(channel)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
@ -1015,11 +1137,19 @@ func (s *Service) startChannelInternal(channel *IMChannel, factory AdapterFactor
|
|||
func (s *Service) StopChannel(channelID string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.stopLeaderRetryLocked(channelID)
|
||||
if cs, ok := s.channels[channelID]; ok {
|
||||
s.stopChannelLocked(channelID, cs)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) stopLeaderRetryLocked(channelID string) {
|
||||
if retry, ok := s.leaderRetries[channelID]; ok {
|
||||
retry.cancel()
|
||||
delete(s.leaderRetries, channelID)
|
||||
}
|
||||
}
|
||||
|
||||
// stopChannelLocked stops a channel and removes it from the map.
|
||||
// Caller must hold s.mu.
|
||||
func (s *Service) stopChannelLocked(channelID string, cs *channelState) {
|
||||
|
|
@ -1055,8 +1185,8 @@ func (s *Service) tryAcquireWSLeader(channelID string) bool {
|
|||
key := RedisKeyLeader + channelID
|
||||
ok, err := s.redis.SetNX(context.Background(), key, s.instanceID, wsLeaderTTL).Result()
|
||||
if err != nil {
|
||||
logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v, assuming leader", channelID, err)
|
||||
return true // Redis error: proceed anyway to avoid channel getting stuck
|
||||
logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v; connection will retry without taking leadership", channelID, err)
|
||||
return false
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
|
@ -1128,17 +1258,58 @@ func (s *Service) wsLeaderRenewLoop(ctx context.Context, channelID string) {
|
|||
s.StopChannel(channelID)
|
||||
return
|
||||
}
|
||||
_, cached, running := s.GetChannelAdapter(channelID)
|
||||
if !running {
|
||||
return
|
||||
}
|
||||
if !sameChannelRuntimeConfig(cached, ch) {
|
||||
logger.Infof(context.Background(),
|
||||
"[IM] Channel %s config changed; rebuilding runtime", channelID)
|
||||
// StartChannel synchronously stops the old runtime, releases its
|
||||
// lease, and then competes to start the fresh configuration. Return
|
||||
// because the old renewal context has been cancelled.
|
||||
if err := s.StartChannel(ch); err != nil && !s.stopped.Load() {
|
||||
logger.Warnf(context.Background(),
|
||||
"[IM] Rebuild changed channel %s failed: %v", channelID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) scheduleWSLeaderRetry(channel *IMChannel) {
|
||||
retryCtx, cancel := context.WithCancel(context.Background())
|
||||
state := &leaderRetryState{cancel: cancel}
|
||||
s.mu.Lock()
|
||||
if existing, ok := s.leaderRetries[channel.ID]; ok {
|
||||
existing.cancel()
|
||||
}
|
||||
if s.stopped.Load() {
|
||||
s.mu.Unlock()
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
s.leaderRetries[channel.ID] = state
|
||||
s.mu.Unlock()
|
||||
go s.wsLeaderRetryLoop(retryCtx, channel, state)
|
||||
}
|
||||
|
||||
// wsLeaderRetryLoop periodically tries to acquire the WebSocket leader lock.
|
||||
// When it succeeds, it starts the channel adapter.
|
||||
func (s *Service) wsLeaderRetryLoop(channel *IMChannel) {
|
||||
// When it succeeds, it starts the channel adapter. A per-channel retry state
|
||||
// ensures repeated config events cannot accumulate duplicate retry goroutines.
|
||||
func (s *Service) wsLeaderRetryLoop(ctx context.Context, channel *IMChannel, state *leaderRetryState) {
|
||||
ticker := time.NewTicker(wsLeaderRetryInterval)
|
||||
defer ticker.Stop()
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
if s.leaderRetries[channel.ID] == state {
|
||||
delete(s.leaderRetries, channel.ID)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
|
|
@ -1190,6 +1361,8 @@ func (s *Service) wsLeaderRetryLoop(channel *IMChannel) {
|
|||
}
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-s.stopCh:
|
||||
return
|
||||
}
|
||||
|
|
@ -1317,6 +1490,68 @@ func (s *Service) GetChannelAdapter(channelID string) (Adapter, *IMChannel, bool
|
|||
return cs.Adapter, cs.Channel, true
|
||||
}
|
||||
|
||||
// EnsureChannelAdapter loads the durable channel row before serving a webhook
|
||||
// callback. This prevents a replica that missed an invalidation event from
|
||||
// verifying or processing the callback with stale credentials/configuration.
|
||||
func (s *Service) EnsureChannelAdapter(channelID string) (Adapter, *IMChannel, error) {
|
||||
fresh, err := s.GetChannelByID(channelID)
|
||||
if err != nil {
|
||||
// A deleted channel may still exist in this replica's runtime map. Do
|
||||
// not tear down a healthy runtime on a transient database failure.
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
s.StopChannel(channelID)
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
if !fresh.Enabled {
|
||||
s.StopChannel(channelID)
|
||||
return nil, fresh, fmt.Errorf("channel is disabled")
|
||||
}
|
||||
|
||||
adapter, cached, ok := s.GetChannelAdapter(channelID)
|
||||
if !ok || !sameChannelRuntimeConfig(cached, fresh) {
|
||||
if err := s.StartChannel(fresh); err != nil {
|
||||
return nil, fresh, err
|
||||
}
|
||||
adapter, cached, ok = s.GetChannelAdapter(channelID)
|
||||
if !ok {
|
||||
return nil, fresh, fmt.Errorf("channel adapter is not active on this instance")
|
||||
}
|
||||
}
|
||||
return adapter, cached, nil
|
||||
}
|
||||
|
||||
// sameChannelRuntimeConfig compares every field that affects adapter creation
|
||||
// or message routing. It intentionally does not compare UpdatedAt: PostgreSQL
|
||||
// timestamp precision can differ from the in-memory value assigned by GORM,
|
||||
// which would otherwise rebuild webhook adapters on every callback.
|
||||
func sameChannelRuntimeConfig(cached, fresh *IMChannel) bool {
|
||||
if cached == nil || fresh == nil {
|
||||
return false
|
||||
}
|
||||
return cached.ID == fresh.ID &&
|
||||
cached.TenantID == fresh.TenantID &&
|
||||
cached.AgentID == fresh.AgentID &&
|
||||
cached.Platform == fresh.Platform &&
|
||||
cached.Enabled == fresh.Enabled &&
|
||||
cached.Mode == fresh.Mode &&
|
||||
cached.OutputMode == fresh.OutputMode &&
|
||||
cached.KnowledgeBaseID == fresh.KnowledgeBaseID &&
|
||||
cached.SessionMode == fresh.SessionMode &&
|
||||
equalChannelCredentials(cached.Credentials, fresh.Credentials)
|
||||
}
|
||||
|
||||
func equalChannelCredentials(left, right types.JSON) bool {
|
||||
var leftValue, rightValue any
|
||||
if err := json.Unmarshal(left, &leftValue); err != nil {
|
||||
return string(left) == string(right)
|
||||
}
|
||||
if err := json.Unmarshal(right, &rightValue); err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(leftValue, rightValue)
|
||||
}
|
||||
|
||||
// GetChannelByID loads a channel from the database.
|
||||
func (s *Service) GetChannelByID(channelID string) (*IMChannel, error) {
|
||||
var ch IMChannel
|
||||
|
|
@ -2739,6 +2974,7 @@ func (s *Service) CreateChannel(channel *IMChannel) error {
|
|||
logger.Warnf(context.Background(), "[IM] Created channel %s but failed to start: %v", channel.ID, err)
|
||||
}
|
||||
}
|
||||
s.publishChannelConfigChange(channel.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -2775,6 +3011,7 @@ func (s *Service) UpdateChannel(channel *IMChannel) error {
|
|||
logger.Warnf(context.Background(), "[IM] Updated channel %s but failed to restart: %v", channel.ID, err)
|
||||
}
|
||||
}
|
||||
s.publishChannelConfigChange(channel.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -2787,19 +3024,22 @@ func (s *Service) DeleteChannelsByAgent(agentID string, tenantID uint64) error {
|
|||
Find(&channels).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range channels {
|
||||
s.StopChannel(channels[i].ID)
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.db.Where("agent_id = ? AND tenant_id = ? AND deleted_at IS NULL", agentID, tenantID).
|
||||
Delete(&IMChannel{}).Error
|
||||
if err := s.db.Where("agent_id = ? AND tenant_id = ? AND deleted_at IS NULL", agentID, tenantID).
|
||||
Delete(&IMChannel{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range channels {
|
||||
s.StopChannel(channels[i].ID)
|
||||
s.publishChannelConfigChange(channels[i].ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteChannel soft-deletes a channel and stops it. Only deletes if the channel belongs to the given tenant.
|
||||
func (s *Service) DeleteChannel(channelID string, tenantID uint64) error {
|
||||
s.StopChannel(channelID)
|
||||
result := s.db.Where("id = ? AND tenant_id = ?", channelID, tenantID).Delete(&IMChannel{})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
|
|
@ -2807,6 +3047,8 @@ func (s *Service) DeleteChannel(channelID string, tenantID uint64) error {
|
|||
if result.RowsAffected == 0 {
|
||||
return fmt.Errorf("channel not found")
|
||||
}
|
||||
s.StopChannel(channelID)
|
||||
s.publishChannelConfigChange(channelID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -2827,6 +3069,7 @@ func (s *Service) ToggleChannel(channelID string, tenantID uint64) (*IMChannel,
|
|||
} else {
|
||||
s.StopChannel(channelID)
|
||||
}
|
||||
s.publishChannelConfigChange(channelID)
|
||||
return &ch, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,336 @@
|
|||
package im
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type lifecycleTestAdapter struct{}
|
||||
|
||||
func (*lifecycleTestAdapter) Platform() Platform { return Platform("test") }
|
||||
func (*lifecycleTestAdapter) VerifyCallback(*gin.Context) error { return nil }
|
||||
func (*lifecycleTestAdapter) ParseCallback(*gin.Context) (*IncomingMessage, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (*lifecycleTestAdapter) SendReply(context.Context, *IncomingMessage, *ReplyMessage) error {
|
||||
return nil
|
||||
}
|
||||
func (*lifecycleTestAdapter) HandleURLVerification(*gin.Context) bool { return false }
|
||||
|
||||
type lifecycleFactoryCounters struct {
|
||||
starts atomic.Int32
|
||||
stops atomic.Int32
|
||||
}
|
||||
|
||||
func (c *lifecycleFactoryCounters) factory() AdapterFactory {
|
||||
return func(context.Context, *IMChannel, func(context.Context, *IncomingMessage) error) (Adapter, context.CancelFunc, error) {
|
||||
c.starts.Add(1)
|
||||
var once sync.Once
|
||||
return &lifecycleTestAdapter{}, func() {
|
||||
once.Do(func() { c.stops.Add(1) })
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func newLifecycleTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:im-lifecycle-%d?mode=memory&cache=shared", time.Now().UnixNano())), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
// IMChannel's production schema uses PostgreSQL's uuid_generate_v4()
|
||||
// default, which SQLite cannot parse. Keep an equivalent minimal table for
|
||||
// lifecycle tests; IDs are assigned explicitly below.
|
||||
if err := db.Exec(`CREATE TABLE im_channels (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
agent_id TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
enabled NUMERIC NOT NULL DEFAULT 1,
|
||||
mode TEXT NOT NULL DEFAULT 'websocket',
|
||||
output_mode TEXT NOT NULL DEFAULT 'stream',
|
||||
knowledge_base_id TEXT DEFAULT '',
|
||||
bot_identity TEXT NOT NULL DEFAULT '',
|
||||
session_mode TEXT NOT NULL DEFAULT 'user',
|
||||
credentials TEXT NOT NULL DEFAULT '{}',
|
||||
created_at DATETIME,
|
||||
updated_at DATETIME,
|
||||
deleted_at DATETIME
|
||||
)`).Error; err != nil {
|
||||
t.Fatalf("create im_channels: %v", err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func newLifecycleTestService(db *gorm.DB, redisClient *redis.Client, instanceID string) *Service {
|
||||
return &Service{
|
||||
db: db,
|
||||
channels: make(map[string]*channelState),
|
||||
leaderRetries: make(map[string]*leaderRetryState),
|
||||
adapterFactories: make(map[string]AdapterFactory),
|
||||
redis: redisClient,
|
||||
instanceID: instanceID,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func createLifecycleChannel(t *testing.T, db *gorm.DB, id, agentID string) *IMChannel {
|
||||
t.Helper()
|
||||
channel := &IMChannel{
|
||||
ID: id,
|
||||
TenantID: 1,
|
||||
AgentID: agentID,
|
||||
Platform: "test",
|
||||
Enabled: true,
|
||||
Mode: "webhook",
|
||||
OutputMode: "full",
|
||||
SessionMode: string(SessionModeUser),
|
||||
Credentials: types.JSON(`{"token":"v1"}`),
|
||||
}
|
||||
if err := db.Create(channel).Error; err != nil {
|
||||
t.Fatalf("create channel: %v", err)
|
||||
}
|
||||
return channel
|
||||
}
|
||||
|
||||
func TestEnsureChannelAdapterRefreshesStaleConfig(t *testing.T) {
|
||||
db := newLifecycleTestDB(t)
|
||||
channel := createLifecycleChannel(t, db, "channel-refresh", "agent-old")
|
||||
counters := &lifecycleFactoryCounters{}
|
||||
svc := newLifecycleTestService(db, nil, "instance-one")
|
||||
svc.RegisterAdapterFactory("test", counters.factory())
|
||||
t.Cleanup(svc.Stop)
|
||||
|
||||
if err := svc.StartChannel(channel); err != nil {
|
||||
t.Fatalf("start initial channel: %v", err)
|
||||
}
|
||||
if err := db.Model(&IMChannel{}).Where("id = ?", channel.ID).
|
||||
Updates(map[string]any{"agent_id": "agent-new", "credentials": types.JSON(`{"token":"v2"}`)}).Error; err != nil {
|
||||
t.Fatalf("update durable channel: %v", err)
|
||||
}
|
||||
|
||||
_, fresh, err := svc.EnsureChannelAdapter(channel.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ensure channel adapter: %v", err)
|
||||
}
|
||||
if fresh.AgentID != "agent-new" || string(fresh.Credentials) != `{"token":"v2"}` {
|
||||
t.Fatalf("stale runtime config returned: agent=%q credentials=%s", fresh.AgentID, fresh.Credentials)
|
||||
}
|
||||
if counters.starts.Load() != 2 || counters.stops.Load() != 1 {
|
||||
t.Fatalf("runtime was not rebuilt exactly once: starts=%d stops=%d", counters.starts.Load(), counters.stops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureChannelAdapterStopsDisabledCachedChannel(t *testing.T) {
|
||||
db := newLifecycleTestDB(t)
|
||||
channel := createLifecycleChannel(t, db, "channel-disabled", "agent")
|
||||
counters := &lifecycleFactoryCounters{}
|
||||
svc := newLifecycleTestService(db, nil, "instance-one")
|
||||
svc.RegisterAdapterFactory("test", counters.factory())
|
||||
t.Cleanup(svc.Stop)
|
||||
|
||||
if err := svc.StartChannel(channel); err != nil {
|
||||
t.Fatalf("start channel: %v", err)
|
||||
}
|
||||
if err := db.Model(&IMChannel{}).Where("id = ?", channel.ID).Update("enabled", false).Error; err != nil {
|
||||
t.Fatalf("disable channel: %v", err)
|
||||
}
|
||||
if _, _, err := svc.EnsureChannelAdapter(channel.ID); err == nil {
|
||||
t.Fatal("EnsureChannelAdapter() expected disabled error")
|
||||
}
|
||||
if _, _, ok := svc.GetChannelAdapter(channel.ID); ok {
|
||||
t.Fatal("disabled channel remained in runtime map")
|
||||
}
|
||||
if counters.stops.Load() != 1 {
|
||||
t.Fatalf("cleanup calls = %d, want 1", counters.stops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureChannelAdapterKeepsRuntimeOnDatabaseFailure(t *testing.T) {
|
||||
db := newLifecycleTestDB(t)
|
||||
channel := createLifecycleChannel(t, db, "channel-db-failure", "agent")
|
||||
counters := &lifecycleFactoryCounters{}
|
||||
svc := newLifecycleTestService(db, nil, "instance-one")
|
||||
svc.RegisterAdapterFactory("test", counters.factory())
|
||||
t.Cleanup(svc.Stop)
|
||||
|
||||
if err := svc.StartChannel(channel); err != nil {
|
||||
t.Fatalf("start channel: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("resolve sql DB: %v", err)
|
||||
}
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Fatalf("close test DB: %v", err)
|
||||
}
|
||||
if _, _, err := svc.EnsureChannelAdapter(channel.ID); err == nil {
|
||||
t.Fatal("EnsureChannelAdapter() expected database error")
|
||||
}
|
||||
if _, _, ok := svc.GetChannelAdapter(channel.ID); !ok {
|
||||
t.Fatal("transient database failure tore down the cached runtime")
|
||||
}
|
||||
if counters.stops.Load() != 0 {
|
||||
t.Fatalf("cleanup calls = %d, want 0 before service shutdown", counters.stops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelConfigEventReloadsOtherReplica(t *testing.T) {
|
||||
db := newLifecycleTestDB(t)
|
||||
channel := createLifecycleChannel(t, db, "channel-pubsub", "agent-old")
|
||||
redisServer := miniredis.RunT(t)
|
||||
redisOne := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
|
||||
redisTwo := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
|
||||
t.Cleanup(func() { _ = redisOne.Close(); _ = redisTwo.Close() })
|
||||
|
||||
countersOne := &lifecycleFactoryCounters{}
|
||||
countersTwo := &lifecycleFactoryCounters{}
|
||||
svcOne := newLifecycleTestService(db, redisOne, "instance-one")
|
||||
svcTwo := newLifecycleTestService(db, redisTwo, "instance-two")
|
||||
svcOne.RegisterAdapterFactory("test", countersOne.factory())
|
||||
svcTwo.RegisterAdapterFactory("test", countersTwo.factory())
|
||||
svcOne.startChannelConfigSubscriber()
|
||||
svcTwo.startChannelConfigSubscriber()
|
||||
t.Cleanup(svcOne.Stop)
|
||||
t.Cleanup(svcTwo.Stop)
|
||||
|
||||
if err := svcOne.StartChannel(channel); err != nil {
|
||||
t.Fatalf("start channel on instance one: %v", err)
|
||||
}
|
||||
copyForTwo := *channel
|
||||
if err := svcTwo.StartChannel(©ForTwo); err != nil {
|
||||
t.Fatalf("start channel on instance two: %v", err)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
counts, err := redisOne.PubSubNumSub(context.Background(), RedisChannelConfig).Result()
|
||||
if err == nil && counts[RedisChannelConfig] == 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
updated := *channel
|
||||
updated.AgentID = "agent-new"
|
||||
updated.Credentials = types.JSON(`{"token":"v2"}`)
|
||||
if err := svcOne.UpdateChannel(&updated); err != nil {
|
||||
t.Fatalf("update channel: %v", err)
|
||||
}
|
||||
|
||||
deadline = time.Now().Add(2 * time.Second)
|
||||
reloaded := false
|
||||
for time.Now().Before(deadline) {
|
||||
_, runtimeChannel, ok := svcTwo.GetChannelAdapter(channel.ID)
|
||||
if ok && runtimeChannel.AgentID == "agent-new" && string(runtimeChannel.Credentials) == `{"token":"v2"}` {
|
||||
if countersTwo.starts.Load() < 2 || countersTwo.stops.Load() < 1 {
|
||||
t.Fatalf("replica config changed without rebuilding runtime: starts=%d stops=%d", countersTwo.starts.Load(), countersTwo.stops.Load())
|
||||
}
|
||||
reloaded = true
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !reloaded {
|
||||
t.Fatal("second replica did not reload the published channel change")
|
||||
}
|
||||
|
||||
if _, err := svcOne.ToggleChannel(channel.ID, channel.TenantID); err != nil {
|
||||
t.Fatalf("disable channel: %v", err)
|
||||
}
|
||||
deadline = time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, _, ok := svcTwo.GetChannelAdapter(channel.ID); !ok {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("second replica did not stop the disabled channel")
|
||||
}
|
||||
|
||||
func TestServiceStopIsIdempotent(t *testing.T) {
|
||||
db := newLifecycleTestDB(t)
|
||||
channel := createLifecycleChannel(t, db, "channel-stop", "agent")
|
||||
counters := &lifecycleFactoryCounters{}
|
||||
svc := newLifecycleTestService(db, nil, "instance-one")
|
||||
svc.RegisterAdapterFactory("test", counters.factory())
|
||||
if err := svc.StartChannel(channel); err != nil {
|
||||
t.Fatalf("start channel: %v", err)
|
||||
}
|
||||
|
||||
svc.Stop()
|
||||
svc.Stop()
|
||||
|
||||
if counters.stops.Load() != 1 {
|
||||
t.Fatalf("cleanup calls = %d, want exactly 1", counters.stops.Load())
|
||||
}
|
||||
if err := svc.StartChannel(channel); err == nil {
|
||||
t.Fatal("StartChannel() succeeded after service shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameChannelRuntimeConfigUsesSemanticCredentials(t *testing.T) {
|
||||
now := time.Now()
|
||||
cached := &IMChannel{
|
||||
ID: "channel",
|
||||
TenantID: 1,
|
||||
AgentID: "agent",
|
||||
Platform: "test",
|
||||
Enabled: true,
|
||||
Mode: "webhook",
|
||||
OutputMode: "full",
|
||||
SessionMode: string(SessionModeUser),
|
||||
Credentials: types.JSON(`{"token":"secret","timeout":10}`),
|
||||
UpdatedAt: now,
|
||||
}
|
||||
fresh := *cached
|
||||
fresh.Credentials = types.JSON(`{"timeout":10,"token":"secret"}`)
|
||||
fresh.UpdatedAt = now.Truncate(time.Microsecond)
|
||||
|
||||
if !sameChannelRuntimeConfig(cached, &fresh) {
|
||||
t.Fatal("semantic-equivalent credentials or timestamp precision triggered a rebuild")
|
||||
}
|
||||
fresh.Credentials = types.JSON(`{"timeout":10,"token":"changed"}`)
|
||||
if sameChannelRuntimeConfig(cached, &fresh) {
|
||||
t.Fatal("changed credentials did not trigger a rebuild")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaderElectionFailsClosedWhenRedisUnavailable(t *testing.T) {
|
||||
db := newLifecycleTestDB(t)
|
||||
channel := createLifecycleChannel(t, db, "channel-redis-down", "agent")
|
||||
channel.Mode = "websocket"
|
||||
counters := &lifecycleFactoryCounters{}
|
||||
redisClient := redis.NewClient(&redis.Options{Addr: "127.0.0.1:1"})
|
||||
_ = redisClient.Close()
|
||||
svc := newLifecycleTestService(db, redisClient, "instance-one")
|
||||
svc.RegisterAdapterFactory("test", counters.factory())
|
||||
t.Cleanup(svc.Stop)
|
||||
|
||||
if err := svc.StartChannel(channel); err != nil {
|
||||
t.Fatalf("StartChannel() should schedule a retry, got %v", err)
|
||||
}
|
||||
if err := svc.StartChannel(channel); err != nil {
|
||||
t.Fatalf("second StartChannel() should replace the retry, got %v", err)
|
||||
}
|
||||
if counters.starts.Load() != 0 {
|
||||
t.Fatalf("factory starts = %d, want 0 while leader election is unavailable", counters.starts.Load())
|
||||
}
|
||||
svc.mu.RLock()
|
||||
retryCount := len(svc.leaderRetries)
|
||||
svc.mu.RUnlock()
|
||||
if retryCount != 1 {
|
||||
t.Fatalf("leader retry goroutines = %d, want one per channel", retryCount)
|
||||
}
|
||||
}
|
||||
|
|
@ -63,7 +63,14 @@ func NewFactory() im.AdapterFactory {
|
|||
}()
|
||||
|
||||
adapter := NewWSAdapter(client)
|
||||
return adapter, wsCancel, nil
|
||||
// Cancelling the context alone does not unblock gorilla/websocket's
|
||||
// ReadMessage. Close the socket first so disabling, reconfiguring, or
|
||||
// shutting down a channel stops message delivery synchronously.
|
||||
stop := func() {
|
||||
client.Stop()
|
||||
wsCancel()
|
||||
}
|
||||
return adapter, stop, nil
|
||||
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unknown WeCom mode: %s", mode)
|
||||
|
|
|
|||
Loading…
Reference in New Issue