diff --git a/internal/container/container.go b/internal/container/container.go index 6a847379..980437e5 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -378,7 +378,7 @@ func BuildContainer(container *dig.Container) *dig.Container { // IM integration logger.Debugf(ctx, "[Container] Registering IM integration...") must(container.Provide(imPkg.NewService)) - must(container.Invoke(registerIMAdapterFactories)) + must(container.Invoke(registerIMService)) must(container.Provide(handler.NewIMHandler)) must(container.Provide(handler.NewEmbedChannelHandler)) must(container.Provide(handler.NewWeKnoraCloudHandler)) @@ -1542,10 +1542,10 @@ func registerWebSearchProviders(registry *infra_web_search.Registry) { registry.Register("zhipu", infra_web_search.NewZhipuProvider) } -// registerIMAdapterFactories registers adapter factories for each IM platform -// and loads enabled channels from the database. Each platform's factory lives -// in its own subpackage to keep this file focused on wiring. -func registerIMAdapterFactories(imService *imPkg.Service) { +// registerIMService registers adapter factories, loads enabled channels, and +// wires the process-lifetime shutdown hook. Each platform's factory lives in +// its own subpackage to keep this file focused on wiring. +func registerIMService(imService *imPkg.Service, cleaner interfaces.ResourceCleaner) { imService.RegisterAdapterFactory("wecom", wecom.NewFactory()) imService.RegisterAdapterFactory("feishu", feishu.NewFactory(feishu.RegionFeishu)) // Lark is Feishu's international cloud: same adapter, different host/tenant. @@ -1562,6 +1562,11 @@ func registerIMAdapterFactories(imService *imPkg.Service) { if err := imService.LoadAndStartChannels(); err != nil { logger.Warnf(context.Background(), "[IM] Failed to load channels from database: %v", err) } + + cleaner.RegisterWithName("IMService", func() error { + imService.Stop() + return nil + }) } // initConnectorRegistry creates and populates the connector registry with all available connectors. diff --git a/internal/handler/im.go b/internal/handler/im.go index 21917e44..d04818a5 100644 --- a/internal/handler/im.go +++ b/internal/handler/im.go @@ -2,6 +2,7 @@ package handler import ( "context" + "errors" "net/http" "sort" "strings" @@ -10,6 +11,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/gin-gonic/gin" + "gorm.io/gorm" ) // validIMPlatforms is the set of supported IM platforms. @@ -364,29 +366,18 @@ func (h *IMHandler) IMCallback(c *gin.Context) { ctx := c.Request.Context() channelID := c.Param("channel_id") - adapter, channel, ok := h.imService.GetChannelAdapter(channelID) - if !ok { - // Try loading from DB - ch, err := h.imService.GetChannelByID(channelID) - if err != nil { + // Always validate the durable row before using a cached webhook adapter. + // This is the correctness fallback when a replica missed Redis invalidation + // while disconnected, and prevents stale credentials/config from being used. + adapter, channel, err := h.imService.EnsureChannelAdapter(channelID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { logger.Errorf(ctx, "[IM] Channel not found for callback: %s", channelID) c.JSON(http.StatusNotFound, gin.H{"error": "channel not found"}) return } - if err := h.imService.StartChannel(ch); err != nil { - logger.Errorf(ctx, "[IM] Failed to start channel for callback: %v", err) - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"}) - return - } - adapter, channel, ok = h.imService.GetChannelAdapter(channelID) - if !ok { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"}) - return - } - } - - if !channel.Enabled { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel is disabled"}) + logger.Errorf(ctx, "[IM] Channel unavailable for callback %s: %v", channelID, err) + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"}) return } diff --git a/internal/im/service.go b/internal/im/service.go index 62ee338c..26eb0230 100644 --- a/internal/im/service.go +++ b/internal/im/service.go @@ -3,15 +3,18 @@ package im import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" "mime/multipart" "net/textproto" "os" + "reflect" "regexp" "strings" "sync" + "sync/atomic" "time" agenttools "github.com/Tencent/WeKnora/internal/agent/tools" @@ -342,6 +345,9 @@ const ( RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter + // RedisChannelConfig broadcasts durable im_channels mutations so every + // application replica invalidates its local adapter/config snapshot. + RedisChannelConfig = "im:channel:config" defaultRateLimitWindow = 60 * time.Second defaultRateLimitMaxRequests = 10 @@ -355,6 +361,15 @@ type channelState struct { leaderCancel context.CancelFunc // stops the leader renewal goroutine (nil if not leader) } +type leaderRetryState struct { + cancel context.CancelFunc +} + +type channelConfigEvent struct { + ChannelID string `json:"channel_id"` + SourceInstance string `json:"source_instance"` +} + // AdapterFactory creates an Adapter from an IMChannel configuration. // The second return value is an optional cleanup function (e.g., for stopping websocket connections). type AdapterFactory func(ctx context.Context, channel *IMChannel, msgHandler func(ctx context.Context, msg *IncomingMessage) error) (Adapter, context.CancelFunc, error) @@ -407,8 +422,9 @@ type Service struct { cmdRegistry *CommandRegistry // channels maps channel ID -> running channel state - channels map[string]*channelState - mu sync.RWMutex + channels map[string]*channelState + leaderRetries map[string]*leaderRetryState + mu sync.RWMutex // adapterFactories maps platform name -> factory function adapterFactories map[string]AdapterFactory @@ -438,7 +454,10 @@ type Service struct { // instanceID uniquely identifies this service instance for leader election. instanceID string - stopCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once + subscriberOnce sync.Once + stopped atomic.Bool } // makeUserKey builds the canonical key used to identify a user's request @@ -843,6 +862,7 @@ func NewService( oauthManager: oauthManager, cmdRegistry: registry, channels: make(map[string]*channelState), + leaderRetries: make(map[string]*leaderRetryState), adapterFactories: make(map[string]AdapterFactory), rateLimiter: ratelimit.New(redisClient, RedisKeyRateLimit, rlWindow, instanceID), rateLimitMax: rlMax, @@ -887,13 +907,22 @@ func (s *Service) RegisterAdapterFactory(platform string, factory AdapterFactory // Stop gracefully shuts down the service, stopping all channels and background goroutines. func (s *Service) Stop() { - close(s.stopCh) - s.qaQueue.Stop() - s.mu.Lock() - defer s.mu.Unlock() - for id, cs := range s.channels { - s.stopChannelLocked(id, cs) - } + s.stopOnce.Do(func() { + s.stopped.Store(true) + close(s.stopCh) + if s.qaQueue != nil { + s.qaQueue.Stop() + } + s.mu.Lock() + defer s.mu.Unlock() + for id, cs := range s.channels { + s.stopChannelLocked(id, cs) + } + for id, retry := range s.leaderRetries { + retry.cancel() + delete(s.leaderRetries, id) + } + }) } // dedupCleanupLoop periodically cleans up expired entries from the dedup map. @@ -918,6 +947,8 @@ func (s *Service) dedupCleanupLoop() { // LoadAndStartChannels loads all enabled channels from the database and starts them. func (s *Service) LoadAndStartChannels() error { + s.startChannelConfigSubscriber() + ctx := context.Background() var channels []IMChannel if err := s.db.Where("enabled = ? AND deleted_at IS NULL", true).Find(&channels).Error; err != nil { @@ -928,8 +959,13 @@ func (s *Service) LoadAndStartChannels() error { ch := channels[i] if err := s.StartChannel(&ch); err != nil { logger.Warnf(ctx, "[IM] Failed to start channel %s (%s/%s): %v", ch.ID, ch.Platform, ch.Name, err) + } else if _, _, active := s.GetChannelAdapter(ch.ID); active { + // Adapter initialization can launch an asynchronous connection attempt. + // Do not claim network readiness here; platform connection logs report it. + logger.Infof(ctx, "[IM] Initialized channel runtime: id=%s platform=%s name=%s mode=%s agent=%s", + ch.ID, ch.Platform, ch.Name, ch.Mode, ch.AgentID) } else { - logger.Infof(ctx, "[IM] Started channel: id=%s platform=%s name=%s mode=%s agent=%s", + logger.Infof(ctx, "[IM] Channel runtime is on standby: id=%s platform=%s name=%s mode=%s agent=%s", ch.ID, ch.Platform, ch.Name, ch.Mode, ch.AgentID) } } @@ -938,12 +974,98 @@ func (s *Service) LoadAndStartChannels() error { return nil } +// startChannelConfigSubscriber listens for durable channel mutations made by +// other application replicas. Redis Pub/Sub provides the fast path; callback +// freshness checks and the leader renewal DB check remain the durable fallback +// if an event is missed while a replica is disconnected. +func (s *Service) startChannelConfigSubscriber() { + if s.redis == nil { + return + } + s.subscriberOnce.Do(func() { + go s.channelConfigSubscriberLoop() + }) +} + +func (s *Service) channelConfigSubscriberLoop() { + pubsub := s.redis.Subscribe(context.Background(), RedisChannelConfig) + defer pubsub.Close() + + messages := pubsub.Channel() + for { + select { + case <-s.stopCh: + return + case msg, ok := <-messages: + if !ok { + return + } + var event channelConfigEvent + if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil { + logger.Warnf(context.Background(), "[IM] Ignore invalid channel config event: %v", err) + continue + } + if event.ChannelID == "" || event.SourceInstance == s.instanceID { + continue + } + s.reloadChannelFromDB(event.ChannelID, "config event") + } + } +} + +func (s *Service) publishChannelConfigChange(channelID string) { + if s.redis == nil || channelID == "" || s.stopped.Load() { + return + } + payload, err := json.Marshal(channelConfigEvent{ + ChannelID: channelID, + SourceInstance: s.instanceID, + }) + if err != nil { + return + } + if err := s.redis.Publish(context.Background(), RedisChannelConfig, payload).Err(); err != nil { + // The database is the source of truth. Subscribers also verify freshness + // on webhook callbacks / leader renewal, so publication is best-effort. + logger.Warnf(context.Background(), "[IM] Publish channel config event failed for %s: %v", channelID, err) + } +} + +func (s *Service) reloadChannelFromDB(channelID, reason string) { + fresh, err := s.GetChannelByID(channelID) + if errors.Is(err, gorm.ErrRecordNotFound) { + s.StopChannel(channelID) + return + } + if err != nil { + logger.Warnf(context.Background(), "[IM] Reload channel %s after %s failed: %v", channelID, reason, err) + return + } + if !fresh.Enabled { + s.StopChannel(channelID) + return + } + if _, cached, running := s.GetChannelAdapter(channelID); running && sameChannelRuntimeConfig(cached, fresh) { + return + } + + logger.Infof(context.Background(), "[IM] Reloading channel %s after %s", channelID, reason) + if err := s.StartChannel(fresh); err != nil && !s.stopped.Load() { + logger.Warnf(context.Background(), "[IM] Reload channel %s after %s failed: %v", channelID, reason, err) + } +} + // StartChannel creates and registers an adapter for the given channel. // For WebSocket channels with Redis available, only one instance acquires // the leader lock and opens the connection; other instances periodically // retry so they can take over if the leader dies. func (s *Service) StartChannel(channel *IMChannel) error { + if s.stopped.Load() { + return fmt.Errorf("im service is stopped") + } + s.mu.Lock() + s.stopLeaderRetryLocked(channel.ID) factory, ok := s.adapterFactories[channel.Platform] if !ok { s.mu.Unlock() @@ -963,7 +1085,7 @@ func (s *Service) StartChannel(channel *IMChannel) error { if !acquired { logger.Infof(context.Background(), "[IM] Channel %s %s owned by another instance, will retry", channel.ID, channel.Mode) - go s.wsLeaderRetryLoop(channel) + s.scheduleWSLeaderRetry(channel) return nil } } @@ -1015,11 +1137,19 @@ func (s *Service) startChannelInternal(channel *IMChannel, factory AdapterFactor func (s *Service) StopChannel(channelID string) { s.mu.Lock() defer s.mu.Unlock() + s.stopLeaderRetryLocked(channelID) if cs, ok := s.channels[channelID]; ok { s.stopChannelLocked(channelID, cs) } } +func (s *Service) stopLeaderRetryLocked(channelID string) { + if retry, ok := s.leaderRetries[channelID]; ok { + retry.cancel() + delete(s.leaderRetries, channelID) + } +} + // stopChannelLocked stops a channel and removes it from the map. // Caller must hold s.mu. func (s *Service) stopChannelLocked(channelID string, cs *channelState) { @@ -1055,8 +1185,8 @@ func (s *Service) tryAcquireWSLeader(channelID string) bool { key := RedisKeyLeader + channelID ok, err := s.redis.SetNX(context.Background(), key, s.instanceID, wsLeaderTTL).Result() if err != nil { - logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v, assuming leader", channelID, err) - return true // Redis error: proceed anyway to avoid channel getting stuck + logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v; connection will retry without taking leadership", channelID, err) + return false } return ok } @@ -1128,17 +1258,58 @@ func (s *Service) wsLeaderRenewLoop(ctx context.Context, channelID string) { s.StopChannel(channelID) return } + _, cached, running := s.GetChannelAdapter(channelID) + if !running { + return + } + if !sameChannelRuntimeConfig(cached, ch) { + logger.Infof(context.Background(), + "[IM] Channel %s config changed; rebuilding runtime", channelID) + // StartChannel synchronously stops the old runtime, releases its + // lease, and then competes to start the fresh configuration. Return + // because the old renewal context has been cancelled. + if err := s.StartChannel(ch); err != nil && !s.stopped.Load() { + logger.Warnf(context.Background(), + "[IM] Rebuild changed channel %s failed: %v", channelID, err) + } + return + } case <-ctx.Done(): return } } } +func (s *Service) scheduleWSLeaderRetry(channel *IMChannel) { + retryCtx, cancel := context.WithCancel(context.Background()) + state := &leaderRetryState{cancel: cancel} + s.mu.Lock() + if existing, ok := s.leaderRetries[channel.ID]; ok { + existing.cancel() + } + if s.stopped.Load() { + s.mu.Unlock() + cancel() + return + } + s.leaderRetries[channel.ID] = state + s.mu.Unlock() + go s.wsLeaderRetryLoop(retryCtx, channel, state) +} + // wsLeaderRetryLoop periodically tries to acquire the WebSocket leader lock. -// When it succeeds, it starts the channel adapter. -func (s *Service) wsLeaderRetryLoop(channel *IMChannel) { +// When it succeeds, it starts the channel adapter. A per-channel retry state +// ensures repeated config events cannot accumulate duplicate retry goroutines. +func (s *Service) wsLeaderRetryLoop(ctx context.Context, channel *IMChannel, state *leaderRetryState) { ticker := time.NewTicker(wsLeaderRetryInterval) defer ticker.Stop() + defer func() { + s.mu.Lock() + if s.leaderRetries[channel.ID] == state { + delete(s.leaderRetries, channel.ID) + } + s.mu.Unlock() + }() for { select { @@ -1190,6 +1361,8 @@ func (s *Service) wsLeaderRetryLoop(channel *IMChannel) { } return } + case <-ctx.Done(): + return case <-s.stopCh: return } @@ -1317,6 +1490,68 @@ func (s *Service) GetChannelAdapter(channelID string) (Adapter, *IMChannel, bool return cs.Adapter, cs.Channel, true } +// EnsureChannelAdapter loads the durable channel row before serving a webhook +// callback. This prevents a replica that missed an invalidation event from +// verifying or processing the callback with stale credentials/configuration. +func (s *Service) EnsureChannelAdapter(channelID string) (Adapter, *IMChannel, error) { + fresh, err := s.GetChannelByID(channelID) + if err != nil { + // A deleted channel may still exist in this replica's runtime map. Do + // not tear down a healthy runtime on a transient database failure. + if errors.Is(err, gorm.ErrRecordNotFound) { + s.StopChannel(channelID) + } + return nil, nil, err + } + if !fresh.Enabled { + s.StopChannel(channelID) + return nil, fresh, fmt.Errorf("channel is disabled") + } + + adapter, cached, ok := s.GetChannelAdapter(channelID) + if !ok || !sameChannelRuntimeConfig(cached, fresh) { + if err := s.StartChannel(fresh); err != nil { + return nil, fresh, err + } + adapter, cached, ok = s.GetChannelAdapter(channelID) + if !ok { + return nil, fresh, fmt.Errorf("channel adapter is not active on this instance") + } + } + return adapter, cached, nil +} + +// sameChannelRuntimeConfig compares every field that affects adapter creation +// or message routing. It intentionally does not compare UpdatedAt: PostgreSQL +// timestamp precision can differ from the in-memory value assigned by GORM, +// which would otherwise rebuild webhook adapters on every callback. +func sameChannelRuntimeConfig(cached, fresh *IMChannel) bool { + if cached == nil || fresh == nil { + return false + } + return cached.ID == fresh.ID && + cached.TenantID == fresh.TenantID && + cached.AgentID == fresh.AgentID && + cached.Platform == fresh.Platform && + cached.Enabled == fresh.Enabled && + cached.Mode == fresh.Mode && + cached.OutputMode == fresh.OutputMode && + cached.KnowledgeBaseID == fresh.KnowledgeBaseID && + cached.SessionMode == fresh.SessionMode && + equalChannelCredentials(cached.Credentials, fresh.Credentials) +} + +func equalChannelCredentials(left, right types.JSON) bool { + var leftValue, rightValue any + if err := json.Unmarshal(left, &leftValue); err != nil { + return string(left) == string(right) + } + if err := json.Unmarshal(right, &rightValue); err != nil { + return false + } + return reflect.DeepEqual(leftValue, rightValue) +} + // GetChannelByID loads a channel from the database. func (s *Service) GetChannelByID(channelID string) (*IMChannel, error) { var ch IMChannel @@ -2739,6 +2974,7 @@ func (s *Service) CreateChannel(channel *IMChannel) error { logger.Warnf(context.Background(), "[IM] Created channel %s but failed to start: %v", channel.ID, err) } } + s.publishChannelConfigChange(channel.ID) return nil } @@ -2775,6 +3011,7 @@ func (s *Service) UpdateChannel(channel *IMChannel) error { logger.Warnf(context.Background(), "[IM] Updated channel %s but failed to restart: %v", channel.ID, err) } } + s.publishChannelConfigChange(channel.ID) return nil } @@ -2787,19 +3024,22 @@ func (s *Service) DeleteChannelsByAgent(agentID string, tenantID uint64) error { Find(&channels).Error; err != nil { return err } - for i := range channels { - s.StopChannel(channels[i].ID) - } if len(channels) == 0 { return nil } - return s.db.Where("agent_id = ? AND tenant_id = ? AND deleted_at IS NULL", agentID, tenantID). - Delete(&IMChannel{}).Error + if err := s.db.Where("agent_id = ? AND tenant_id = ? AND deleted_at IS NULL", agentID, tenantID). + Delete(&IMChannel{}).Error; err != nil { + return err + } + for i := range channels { + s.StopChannel(channels[i].ID) + s.publishChannelConfigChange(channels[i].ID) + } + return nil } // DeleteChannel soft-deletes a channel and stops it. Only deletes if the channel belongs to the given tenant. func (s *Service) DeleteChannel(channelID string, tenantID uint64) error { - s.StopChannel(channelID) result := s.db.Where("id = ? AND tenant_id = ?", channelID, tenantID).Delete(&IMChannel{}) if result.Error != nil { return result.Error @@ -2807,6 +3047,8 @@ func (s *Service) DeleteChannel(channelID string, tenantID uint64) error { if result.RowsAffected == 0 { return fmt.Errorf("channel not found") } + s.StopChannel(channelID) + s.publishChannelConfigChange(channelID) return nil } @@ -2827,6 +3069,7 @@ func (s *Service) ToggleChannel(channelID string, tenantID uint64) (*IMChannel, } else { s.StopChannel(channelID) } + s.publishChannelConfigChange(channelID) return &ch, nil } diff --git a/internal/im/service_lifecycle_test.go b/internal/im/service_lifecycle_test.go new file mode 100644 index 00000000..934835e5 --- /dev/null +++ b/internal/im/service_lifecycle_test.go @@ -0,0 +1,336 @@ +package im + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/alicebob/miniredis/v2" + "github.com/gin-gonic/gin" + "github.com/redis/go-redis/v9" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type lifecycleTestAdapter struct{} + +func (*lifecycleTestAdapter) Platform() Platform { return Platform("test") } +func (*lifecycleTestAdapter) VerifyCallback(*gin.Context) error { return nil } +func (*lifecycleTestAdapter) ParseCallback(*gin.Context) (*IncomingMessage, error) { + return nil, nil +} +func (*lifecycleTestAdapter) SendReply(context.Context, *IncomingMessage, *ReplyMessage) error { + return nil +} +func (*lifecycleTestAdapter) HandleURLVerification(*gin.Context) bool { return false } + +type lifecycleFactoryCounters struct { + starts atomic.Int32 + stops atomic.Int32 +} + +func (c *lifecycleFactoryCounters) factory() AdapterFactory { + return func(context.Context, *IMChannel, func(context.Context, *IncomingMessage) error) (Adapter, context.CancelFunc, error) { + c.starts.Add(1) + var once sync.Once + return &lifecycleTestAdapter{}, func() { + once.Do(func() { c.stops.Add(1) }) + }, nil + } +} + +func newLifecycleTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:im-lifecycle-%d?mode=memory&cache=shared", time.Now().UnixNano())), &gorm.Config{}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + // IMChannel's production schema uses PostgreSQL's uuid_generate_v4() + // default, which SQLite cannot parse. Keep an equivalent minimal table for + // lifecycle tests; IDs are assigned explicitly below. + if err := db.Exec(`CREATE TABLE im_channels ( + id TEXT PRIMARY KEY, + tenant_id INTEGER NOT NULL, + agent_id TEXT NOT NULL, + platform TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + enabled NUMERIC NOT NULL DEFAULT 1, + mode TEXT NOT NULL DEFAULT 'websocket', + output_mode TEXT NOT NULL DEFAULT 'stream', + knowledge_base_id TEXT DEFAULT '', + bot_identity TEXT NOT NULL DEFAULT '', + session_mode TEXT NOT NULL DEFAULT 'user', + credentials TEXT NOT NULL DEFAULT '{}', + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME + )`).Error; err != nil { + t.Fatalf("create im_channels: %v", err) + } + return db +} + +func newLifecycleTestService(db *gorm.DB, redisClient *redis.Client, instanceID string) *Service { + return &Service{ + db: db, + channels: make(map[string]*channelState), + leaderRetries: make(map[string]*leaderRetryState), + adapterFactories: make(map[string]AdapterFactory), + redis: redisClient, + instanceID: instanceID, + stopCh: make(chan struct{}), + } +} + +func createLifecycleChannel(t *testing.T, db *gorm.DB, id, agentID string) *IMChannel { + t.Helper() + channel := &IMChannel{ + ID: id, + TenantID: 1, + AgentID: agentID, + Platform: "test", + Enabled: true, + Mode: "webhook", + OutputMode: "full", + SessionMode: string(SessionModeUser), + Credentials: types.JSON(`{"token":"v1"}`), + } + if err := db.Create(channel).Error; err != nil { + t.Fatalf("create channel: %v", err) + } + return channel +} + +func TestEnsureChannelAdapterRefreshesStaleConfig(t *testing.T) { + db := newLifecycleTestDB(t) + channel := createLifecycleChannel(t, db, "channel-refresh", "agent-old") + counters := &lifecycleFactoryCounters{} + svc := newLifecycleTestService(db, nil, "instance-one") + svc.RegisterAdapterFactory("test", counters.factory()) + t.Cleanup(svc.Stop) + + if err := svc.StartChannel(channel); err != nil { + t.Fatalf("start initial channel: %v", err) + } + if err := db.Model(&IMChannel{}).Where("id = ?", channel.ID). + Updates(map[string]any{"agent_id": "agent-new", "credentials": types.JSON(`{"token":"v2"}`)}).Error; err != nil { + t.Fatalf("update durable channel: %v", err) + } + + _, fresh, err := svc.EnsureChannelAdapter(channel.ID) + if err != nil { + t.Fatalf("ensure channel adapter: %v", err) + } + if fresh.AgentID != "agent-new" || string(fresh.Credentials) != `{"token":"v2"}` { + t.Fatalf("stale runtime config returned: agent=%q credentials=%s", fresh.AgentID, fresh.Credentials) + } + if counters.starts.Load() != 2 || counters.stops.Load() != 1 { + t.Fatalf("runtime was not rebuilt exactly once: starts=%d stops=%d", counters.starts.Load(), counters.stops.Load()) + } +} + +func TestEnsureChannelAdapterStopsDisabledCachedChannel(t *testing.T) { + db := newLifecycleTestDB(t) + channel := createLifecycleChannel(t, db, "channel-disabled", "agent") + counters := &lifecycleFactoryCounters{} + svc := newLifecycleTestService(db, nil, "instance-one") + svc.RegisterAdapterFactory("test", counters.factory()) + t.Cleanup(svc.Stop) + + if err := svc.StartChannel(channel); err != nil { + t.Fatalf("start channel: %v", err) + } + if err := db.Model(&IMChannel{}).Where("id = ?", channel.ID).Update("enabled", false).Error; err != nil { + t.Fatalf("disable channel: %v", err) + } + if _, _, err := svc.EnsureChannelAdapter(channel.ID); err == nil { + t.Fatal("EnsureChannelAdapter() expected disabled error") + } + if _, _, ok := svc.GetChannelAdapter(channel.ID); ok { + t.Fatal("disabled channel remained in runtime map") + } + if counters.stops.Load() != 1 { + t.Fatalf("cleanup calls = %d, want 1", counters.stops.Load()) + } +} + +func TestEnsureChannelAdapterKeepsRuntimeOnDatabaseFailure(t *testing.T) { + db := newLifecycleTestDB(t) + channel := createLifecycleChannel(t, db, "channel-db-failure", "agent") + counters := &lifecycleFactoryCounters{} + svc := newLifecycleTestService(db, nil, "instance-one") + svc.RegisterAdapterFactory("test", counters.factory()) + t.Cleanup(svc.Stop) + + if err := svc.StartChannel(channel); err != nil { + t.Fatalf("start channel: %v", err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("resolve sql DB: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close test DB: %v", err) + } + if _, _, err := svc.EnsureChannelAdapter(channel.ID); err == nil { + t.Fatal("EnsureChannelAdapter() expected database error") + } + if _, _, ok := svc.GetChannelAdapter(channel.ID); !ok { + t.Fatal("transient database failure tore down the cached runtime") + } + if counters.stops.Load() != 0 { + t.Fatalf("cleanup calls = %d, want 0 before service shutdown", counters.stops.Load()) + } +} + +func TestChannelConfigEventReloadsOtherReplica(t *testing.T) { + db := newLifecycleTestDB(t) + channel := createLifecycleChannel(t, db, "channel-pubsub", "agent-old") + redisServer := miniredis.RunT(t) + redisOne := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + redisTwo := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + t.Cleanup(func() { _ = redisOne.Close(); _ = redisTwo.Close() }) + + countersOne := &lifecycleFactoryCounters{} + countersTwo := &lifecycleFactoryCounters{} + svcOne := newLifecycleTestService(db, redisOne, "instance-one") + svcTwo := newLifecycleTestService(db, redisTwo, "instance-two") + svcOne.RegisterAdapterFactory("test", countersOne.factory()) + svcTwo.RegisterAdapterFactory("test", countersTwo.factory()) + svcOne.startChannelConfigSubscriber() + svcTwo.startChannelConfigSubscriber() + t.Cleanup(svcOne.Stop) + t.Cleanup(svcTwo.Stop) + + if err := svcOne.StartChannel(channel); err != nil { + t.Fatalf("start channel on instance one: %v", err) + } + copyForTwo := *channel + if err := svcTwo.StartChannel(©ForTwo); err != nil { + t.Fatalf("start channel on instance two: %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + counts, err := redisOne.PubSubNumSub(context.Background(), RedisChannelConfig).Result() + if err == nil && counts[RedisChannelConfig] == 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + updated := *channel + updated.AgentID = "agent-new" + updated.Credentials = types.JSON(`{"token":"v2"}`) + if err := svcOne.UpdateChannel(&updated); err != nil { + t.Fatalf("update channel: %v", err) + } + + deadline = time.Now().Add(2 * time.Second) + reloaded := false + for time.Now().Before(deadline) { + _, runtimeChannel, ok := svcTwo.GetChannelAdapter(channel.ID) + if ok && runtimeChannel.AgentID == "agent-new" && string(runtimeChannel.Credentials) == `{"token":"v2"}` { + if countersTwo.starts.Load() < 2 || countersTwo.stops.Load() < 1 { + t.Fatalf("replica config changed without rebuilding runtime: starts=%d stops=%d", countersTwo.starts.Load(), countersTwo.stops.Load()) + } + reloaded = true + break + } + time.Sleep(10 * time.Millisecond) + } + if !reloaded { + t.Fatal("second replica did not reload the published channel change") + } + + if _, err := svcOne.ToggleChannel(channel.ID, channel.TenantID); err != nil { + t.Fatalf("disable channel: %v", err) + } + deadline = time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, _, ok := svcTwo.GetChannelAdapter(channel.ID); !ok { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("second replica did not stop the disabled channel") +} + +func TestServiceStopIsIdempotent(t *testing.T) { + db := newLifecycleTestDB(t) + channel := createLifecycleChannel(t, db, "channel-stop", "agent") + counters := &lifecycleFactoryCounters{} + svc := newLifecycleTestService(db, nil, "instance-one") + svc.RegisterAdapterFactory("test", counters.factory()) + if err := svc.StartChannel(channel); err != nil { + t.Fatalf("start channel: %v", err) + } + + svc.Stop() + svc.Stop() + + if counters.stops.Load() != 1 { + t.Fatalf("cleanup calls = %d, want exactly 1", counters.stops.Load()) + } + if err := svc.StartChannel(channel); err == nil { + t.Fatal("StartChannel() succeeded after service shutdown") + } +} + +func TestSameChannelRuntimeConfigUsesSemanticCredentials(t *testing.T) { + now := time.Now() + cached := &IMChannel{ + ID: "channel", + TenantID: 1, + AgentID: "agent", + Platform: "test", + Enabled: true, + Mode: "webhook", + OutputMode: "full", + SessionMode: string(SessionModeUser), + Credentials: types.JSON(`{"token":"secret","timeout":10}`), + UpdatedAt: now, + } + fresh := *cached + fresh.Credentials = types.JSON(`{"timeout":10,"token":"secret"}`) + fresh.UpdatedAt = now.Truncate(time.Microsecond) + + if !sameChannelRuntimeConfig(cached, &fresh) { + t.Fatal("semantic-equivalent credentials or timestamp precision triggered a rebuild") + } + fresh.Credentials = types.JSON(`{"timeout":10,"token":"changed"}`) + if sameChannelRuntimeConfig(cached, &fresh) { + t.Fatal("changed credentials did not trigger a rebuild") + } +} + +func TestLeaderElectionFailsClosedWhenRedisUnavailable(t *testing.T) { + db := newLifecycleTestDB(t) + channel := createLifecycleChannel(t, db, "channel-redis-down", "agent") + channel.Mode = "websocket" + counters := &lifecycleFactoryCounters{} + redisClient := redis.NewClient(&redis.Options{Addr: "127.0.0.1:1"}) + _ = redisClient.Close() + svc := newLifecycleTestService(db, redisClient, "instance-one") + svc.RegisterAdapterFactory("test", counters.factory()) + t.Cleanup(svc.Stop) + + if err := svc.StartChannel(channel); err != nil { + t.Fatalf("StartChannel() should schedule a retry, got %v", err) + } + if err := svc.StartChannel(channel); err != nil { + t.Fatalf("second StartChannel() should replace the retry, got %v", err) + } + if counters.starts.Load() != 0 { + t.Fatalf("factory starts = %d, want 0 while leader election is unavailable", counters.starts.Load()) + } + svc.mu.RLock() + retryCount := len(svc.leaderRetries) + svc.mu.RUnlock() + if retryCount != 1 { + t.Fatalf("leader retry goroutines = %d, want one per channel", retryCount) + } +} diff --git a/internal/im/wecom/factory.go b/internal/im/wecom/factory.go index 20be98c1..273df28b 100644 --- a/internal/im/wecom/factory.go +++ b/internal/im/wecom/factory.go @@ -63,7 +63,14 @@ func NewFactory() im.AdapterFactory { }() adapter := NewWSAdapter(client) - return adapter, wsCancel, nil + // Cancelling the context alone does not unblock gorilla/websocket's + // ReadMessage. Close the socket first so disabling, reconfiguring, or + // shutting down a channel stops message delivery synchronously. + stop := func() { + client.Stop() + wsCancel() + } + return adapter, stop, nil default: return nil, nil, fmt.Errorf("unknown WeCom mode: %s", mode)