fix(im): harden connection lifecycle across replicas

main
wizardchen 2026-07-27 20:21:13 +08:00 committed by lyingbug
parent 4826de44de
commit 53ddad57c0
5 changed files with 629 additions and 47 deletions

View File

@ -378,7 +378,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// IM integration
logger.Debugf(ctx, "[Container] Registering IM integration...")
must(container.Provide(imPkg.NewService))
must(container.Invoke(registerIMAdapterFactories))
must(container.Invoke(registerIMService))
must(container.Provide(handler.NewIMHandler))
must(container.Provide(handler.NewEmbedChannelHandler))
must(container.Provide(handler.NewWeKnoraCloudHandler))
@ -1542,10 +1542,10 @@ func registerWebSearchProviders(registry *infra_web_search.Registry) {
registry.Register("zhipu", infra_web_search.NewZhipuProvider)
}
// registerIMAdapterFactories registers adapter factories for each IM platform
// and loads enabled channels from the database. Each platform's factory lives
// in its own subpackage to keep this file focused on wiring.
func registerIMAdapterFactories(imService *imPkg.Service) {
// registerIMService registers adapter factories, loads enabled channels, and
// wires the process-lifetime shutdown hook. Each platform's factory lives in
// its own subpackage to keep this file focused on wiring.
func registerIMService(imService *imPkg.Service, cleaner interfaces.ResourceCleaner) {
imService.RegisterAdapterFactory("wecom", wecom.NewFactory())
imService.RegisterAdapterFactory("feishu", feishu.NewFactory(feishu.RegionFeishu))
// Lark is Feishu's international cloud: same adapter, different host/tenant.
@ -1562,6 +1562,11 @@ func registerIMAdapterFactories(imService *imPkg.Service) {
if err := imService.LoadAndStartChannels(); err != nil {
logger.Warnf(context.Background(), "[IM] Failed to load channels from database: %v", err)
}
cleaner.RegisterWithName("IMService", func() error {
imService.Stop()
return nil
})
}
// initConnectorRegistry creates and populates the connector registry with all available connectors.

View File

@ -2,6 +2,7 @@ package handler
import (
"context"
"errors"
"net/http"
"sort"
"strings"
@ -10,6 +11,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
// validIMPlatforms is the set of supported IM platforms.
@ -364,29 +366,18 @@ func (h *IMHandler) IMCallback(c *gin.Context) {
ctx := c.Request.Context()
channelID := c.Param("channel_id")
adapter, channel, ok := h.imService.GetChannelAdapter(channelID)
if !ok {
// Try loading from DB
ch, err := h.imService.GetChannelByID(channelID)
if err != nil {
// Always validate the durable row before using a cached webhook adapter.
// This is the correctness fallback when a replica missed Redis invalidation
// while disconnected, and prevents stale credentials/config from being used.
adapter, channel, err := h.imService.EnsureChannelAdapter(channelID)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
logger.Errorf(ctx, "[IM] Channel not found for callback: %s", channelID)
c.JSON(http.StatusNotFound, gin.H{"error": "channel not found"})
return
}
if err := h.imService.StartChannel(ch); err != nil {
logger.Errorf(ctx, "[IM] Failed to start channel for callback: %v", err)
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"})
return
}
adapter, channel, ok = h.imService.GetChannelAdapter(channelID)
if !ok {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"})
return
}
}
if !channel.Enabled {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel is disabled"})
logger.Errorf(ctx, "[IM] Channel unavailable for callback %s: %v", channelID, err)
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "channel not available"})
return
}

View File

@ -3,15 +3,18 @@ package im
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/textproto"
"os"
"reflect"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
@ -342,6 +345,9 @@ const (
RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter
RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting
RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter
// RedisChannelConfig broadcasts durable im_channels mutations so every
// application replica invalidates its local adapter/config snapshot.
RedisChannelConfig = "im:channel:config"
defaultRateLimitWindow = 60 * time.Second
defaultRateLimitMaxRequests = 10
@ -355,6 +361,15 @@ type channelState struct {
leaderCancel context.CancelFunc // stops the leader renewal goroutine (nil if not leader)
}
type leaderRetryState struct {
cancel context.CancelFunc
}
type channelConfigEvent struct {
ChannelID string `json:"channel_id"`
SourceInstance string `json:"source_instance"`
}
// AdapterFactory creates an Adapter from an IMChannel configuration.
// The second return value is an optional cleanup function (e.g., for stopping websocket connections).
type AdapterFactory func(ctx context.Context, channel *IMChannel, msgHandler func(ctx context.Context, msg *IncomingMessage) error) (Adapter, context.CancelFunc, error)
@ -407,8 +422,9 @@ type Service struct {
cmdRegistry *CommandRegistry
// channels maps channel ID -> running channel state
channels map[string]*channelState
mu sync.RWMutex
channels map[string]*channelState
leaderRetries map[string]*leaderRetryState
mu sync.RWMutex
// adapterFactories maps platform name -> factory function
adapterFactories map[string]AdapterFactory
@ -438,7 +454,10 @@ type Service struct {
// instanceID uniquely identifies this service instance for leader election.
instanceID string
stopCh chan struct{}
stopCh chan struct{}
stopOnce sync.Once
subscriberOnce sync.Once
stopped atomic.Bool
}
// makeUserKey builds the canonical key used to identify a user's request
@ -843,6 +862,7 @@ func NewService(
oauthManager: oauthManager,
cmdRegistry: registry,
channels: make(map[string]*channelState),
leaderRetries: make(map[string]*leaderRetryState),
adapterFactories: make(map[string]AdapterFactory),
rateLimiter: ratelimit.New(redisClient, RedisKeyRateLimit, rlWindow, instanceID),
rateLimitMax: rlMax,
@ -887,13 +907,22 @@ func (s *Service) RegisterAdapterFactory(platform string, factory AdapterFactory
// Stop gracefully shuts down the service, stopping all channels and background goroutines.
func (s *Service) Stop() {
close(s.stopCh)
s.qaQueue.Stop()
s.mu.Lock()
defer s.mu.Unlock()
for id, cs := range s.channels {
s.stopChannelLocked(id, cs)
}
s.stopOnce.Do(func() {
s.stopped.Store(true)
close(s.stopCh)
if s.qaQueue != nil {
s.qaQueue.Stop()
}
s.mu.Lock()
defer s.mu.Unlock()
for id, cs := range s.channels {
s.stopChannelLocked(id, cs)
}
for id, retry := range s.leaderRetries {
retry.cancel()
delete(s.leaderRetries, id)
}
})
}
// dedupCleanupLoop periodically cleans up expired entries from the dedup map.
@ -918,6 +947,8 @@ func (s *Service) dedupCleanupLoop() {
// LoadAndStartChannels loads all enabled channels from the database and starts them.
func (s *Service) LoadAndStartChannels() error {
s.startChannelConfigSubscriber()
ctx := context.Background()
var channels []IMChannel
if err := s.db.Where("enabled = ? AND deleted_at IS NULL", true).Find(&channels).Error; err != nil {
@ -928,8 +959,13 @@ func (s *Service) LoadAndStartChannels() error {
ch := channels[i]
if err := s.StartChannel(&ch); err != nil {
logger.Warnf(ctx, "[IM] Failed to start channel %s (%s/%s): %v", ch.ID, ch.Platform, ch.Name, err)
} else if _, _, active := s.GetChannelAdapter(ch.ID); active {
// Adapter initialization can launch an asynchronous connection attempt.
// Do not claim network readiness here; platform connection logs report it.
logger.Infof(ctx, "[IM] Initialized channel runtime: id=%s platform=%s name=%s mode=%s agent=%s",
ch.ID, ch.Platform, ch.Name, ch.Mode, ch.AgentID)
} else {
logger.Infof(ctx, "[IM] Started channel: id=%s platform=%s name=%s mode=%s agent=%s",
logger.Infof(ctx, "[IM] Channel runtime is on standby: id=%s platform=%s name=%s mode=%s agent=%s",
ch.ID, ch.Platform, ch.Name, ch.Mode, ch.AgentID)
}
}
@ -938,12 +974,98 @@ func (s *Service) LoadAndStartChannels() error {
return nil
}
// startChannelConfigSubscriber listens for durable channel mutations made by
// other application replicas. Redis Pub/Sub provides the fast path; callback
// freshness checks and the leader renewal DB check remain the durable fallback
// if an event is missed while a replica is disconnected.
func (s *Service) startChannelConfigSubscriber() {
if s.redis == nil {
return
}
s.subscriberOnce.Do(func() {
go s.channelConfigSubscriberLoop()
})
}
func (s *Service) channelConfigSubscriberLoop() {
pubsub := s.redis.Subscribe(context.Background(), RedisChannelConfig)
defer pubsub.Close()
messages := pubsub.Channel()
for {
select {
case <-s.stopCh:
return
case msg, ok := <-messages:
if !ok {
return
}
var event channelConfigEvent
if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil {
logger.Warnf(context.Background(), "[IM] Ignore invalid channel config event: %v", err)
continue
}
if event.ChannelID == "" || event.SourceInstance == s.instanceID {
continue
}
s.reloadChannelFromDB(event.ChannelID, "config event")
}
}
}
func (s *Service) publishChannelConfigChange(channelID string) {
if s.redis == nil || channelID == "" || s.stopped.Load() {
return
}
payload, err := json.Marshal(channelConfigEvent{
ChannelID: channelID,
SourceInstance: s.instanceID,
})
if err != nil {
return
}
if err := s.redis.Publish(context.Background(), RedisChannelConfig, payload).Err(); err != nil {
// The database is the source of truth. Subscribers also verify freshness
// on webhook callbacks / leader renewal, so publication is best-effort.
logger.Warnf(context.Background(), "[IM] Publish channel config event failed for %s: %v", channelID, err)
}
}
func (s *Service) reloadChannelFromDB(channelID, reason string) {
fresh, err := s.GetChannelByID(channelID)
if errors.Is(err, gorm.ErrRecordNotFound) {
s.StopChannel(channelID)
return
}
if err != nil {
logger.Warnf(context.Background(), "[IM] Reload channel %s after %s failed: %v", channelID, reason, err)
return
}
if !fresh.Enabled {
s.StopChannel(channelID)
return
}
if _, cached, running := s.GetChannelAdapter(channelID); running && sameChannelRuntimeConfig(cached, fresh) {
return
}
logger.Infof(context.Background(), "[IM] Reloading channel %s after %s", channelID, reason)
if err := s.StartChannel(fresh); err != nil && !s.stopped.Load() {
logger.Warnf(context.Background(), "[IM] Reload channel %s after %s failed: %v", channelID, reason, err)
}
}
// StartChannel creates and registers an adapter for the given channel.
// For WebSocket channels with Redis available, only one instance acquires
// the leader lock and opens the connection; other instances periodically
// retry so they can take over if the leader dies.
func (s *Service) StartChannel(channel *IMChannel) error {
if s.stopped.Load() {
return fmt.Errorf("im service is stopped")
}
s.mu.Lock()
s.stopLeaderRetryLocked(channel.ID)
factory, ok := s.adapterFactories[channel.Platform]
if !ok {
s.mu.Unlock()
@ -963,7 +1085,7 @@ func (s *Service) StartChannel(channel *IMChannel) error {
if !acquired {
logger.Infof(context.Background(),
"[IM] Channel %s %s owned by another instance, will retry", channel.ID, channel.Mode)
go s.wsLeaderRetryLoop(channel)
s.scheduleWSLeaderRetry(channel)
return nil
}
}
@ -1015,11 +1137,19 @@ func (s *Service) startChannelInternal(channel *IMChannel, factory AdapterFactor
func (s *Service) StopChannel(channelID string) {
s.mu.Lock()
defer s.mu.Unlock()
s.stopLeaderRetryLocked(channelID)
if cs, ok := s.channels[channelID]; ok {
s.stopChannelLocked(channelID, cs)
}
}
func (s *Service) stopLeaderRetryLocked(channelID string) {
if retry, ok := s.leaderRetries[channelID]; ok {
retry.cancel()
delete(s.leaderRetries, channelID)
}
}
// stopChannelLocked stops a channel and removes it from the map.
// Caller must hold s.mu.
func (s *Service) stopChannelLocked(channelID string, cs *channelState) {
@ -1055,8 +1185,8 @@ func (s *Service) tryAcquireWSLeader(channelID string) bool {
key := RedisKeyLeader + channelID
ok, err := s.redis.SetNX(context.Background(), key, s.instanceID, wsLeaderTTL).Result()
if err != nil {
logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v, assuming leader", channelID, err)
return true // Redis error: proceed anyway to avoid channel getting stuck
logger.Warnf(context.Background(), "[IM] Redis leader election failed for %s: %v; connection will retry without taking leadership", channelID, err)
return false
}
return ok
}
@ -1128,17 +1258,58 @@ func (s *Service) wsLeaderRenewLoop(ctx context.Context, channelID string) {
s.StopChannel(channelID)
return
}
_, cached, running := s.GetChannelAdapter(channelID)
if !running {
return
}
if !sameChannelRuntimeConfig(cached, ch) {
logger.Infof(context.Background(),
"[IM] Channel %s config changed; rebuilding runtime", channelID)
// StartChannel synchronously stops the old runtime, releases its
// lease, and then competes to start the fresh configuration. Return
// because the old renewal context has been cancelled.
if err := s.StartChannel(ch); err != nil && !s.stopped.Load() {
logger.Warnf(context.Background(),
"[IM] Rebuild changed channel %s failed: %v", channelID, err)
}
return
}
case <-ctx.Done():
return
}
}
}
func (s *Service) scheduleWSLeaderRetry(channel *IMChannel) {
retryCtx, cancel := context.WithCancel(context.Background())
state := &leaderRetryState{cancel: cancel}
s.mu.Lock()
if existing, ok := s.leaderRetries[channel.ID]; ok {
existing.cancel()
}
if s.stopped.Load() {
s.mu.Unlock()
cancel()
return
}
s.leaderRetries[channel.ID] = state
s.mu.Unlock()
go s.wsLeaderRetryLoop(retryCtx, channel, state)
}
// wsLeaderRetryLoop periodically tries to acquire the WebSocket leader lock.
// When it succeeds, it starts the channel adapter.
func (s *Service) wsLeaderRetryLoop(channel *IMChannel) {
// When it succeeds, it starts the channel adapter. A per-channel retry state
// ensures repeated config events cannot accumulate duplicate retry goroutines.
func (s *Service) wsLeaderRetryLoop(ctx context.Context, channel *IMChannel, state *leaderRetryState) {
ticker := time.NewTicker(wsLeaderRetryInterval)
defer ticker.Stop()
defer func() {
s.mu.Lock()
if s.leaderRetries[channel.ID] == state {
delete(s.leaderRetries, channel.ID)
}
s.mu.Unlock()
}()
for {
select {
@ -1190,6 +1361,8 @@ func (s *Service) wsLeaderRetryLoop(channel *IMChannel) {
}
return
}
case <-ctx.Done():
return
case <-s.stopCh:
return
}
@ -1317,6 +1490,68 @@ func (s *Service) GetChannelAdapter(channelID string) (Adapter, *IMChannel, bool
return cs.Adapter, cs.Channel, true
}
// EnsureChannelAdapter loads the durable channel row before serving a webhook
// callback. This prevents a replica that missed an invalidation event from
// verifying or processing the callback with stale credentials/configuration.
func (s *Service) EnsureChannelAdapter(channelID string) (Adapter, *IMChannel, error) {
fresh, err := s.GetChannelByID(channelID)
if err != nil {
// A deleted channel may still exist in this replica's runtime map. Do
// not tear down a healthy runtime on a transient database failure.
if errors.Is(err, gorm.ErrRecordNotFound) {
s.StopChannel(channelID)
}
return nil, nil, err
}
if !fresh.Enabled {
s.StopChannel(channelID)
return nil, fresh, fmt.Errorf("channel is disabled")
}
adapter, cached, ok := s.GetChannelAdapter(channelID)
if !ok || !sameChannelRuntimeConfig(cached, fresh) {
if err := s.StartChannel(fresh); err != nil {
return nil, fresh, err
}
adapter, cached, ok = s.GetChannelAdapter(channelID)
if !ok {
return nil, fresh, fmt.Errorf("channel adapter is not active on this instance")
}
}
return adapter, cached, nil
}
// sameChannelRuntimeConfig compares every field that affects adapter creation
// or message routing. It intentionally does not compare UpdatedAt: PostgreSQL
// timestamp precision can differ from the in-memory value assigned by GORM,
// which would otherwise rebuild webhook adapters on every callback.
func sameChannelRuntimeConfig(cached, fresh *IMChannel) bool {
if cached == nil || fresh == nil {
return false
}
return cached.ID == fresh.ID &&
cached.TenantID == fresh.TenantID &&
cached.AgentID == fresh.AgentID &&
cached.Platform == fresh.Platform &&
cached.Enabled == fresh.Enabled &&
cached.Mode == fresh.Mode &&
cached.OutputMode == fresh.OutputMode &&
cached.KnowledgeBaseID == fresh.KnowledgeBaseID &&
cached.SessionMode == fresh.SessionMode &&
equalChannelCredentials(cached.Credentials, fresh.Credentials)
}
func equalChannelCredentials(left, right types.JSON) bool {
var leftValue, rightValue any
if err := json.Unmarshal(left, &leftValue); err != nil {
return string(left) == string(right)
}
if err := json.Unmarshal(right, &rightValue); err != nil {
return false
}
return reflect.DeepEqual(leftValue, rightValue)
}
// GetChannelByID loads a channel from the database.
func (s *Service) GetChannelByID(channelID string) (*IMChannel, error) {
var ch IMChannel
@ -2739,6 +2974,7 @@ func (s *Service) CreateChannel(channel *IMChannel) error {
logger.Warnf(context.Background(), "[IM] Created channel %s but failed to start: %v", channel.ID, err)
}
}
s.publishChannelConfigChange(channel.ID)
return nil
}
@ -2775,6 +3011,7 @@ func (s *Service) UpdateChannel(channel *IMChannel) error {
logger.Warnf(context.Background(), "[IM] Updated channel %s but failed to restart: %v", channel.ID, err)
}
}
s.publishChannelConfigChange(channel.ID)
return nil
}
@ -2787,19 +3024,22 @@ func (s *Service) DeleteChannelsByAgent(agentID string, tenantID uint64) error {
Find(&channels).Error; err != nil {
return err
}
for i := range channels {
s.StopChannel(channels[i].ID)
}
if len(channels) == 0 {
return nil
}
return s.db.Where("agent_id = ? AND tenant_id = ? AND deleted_at IS NULL", agentID, tenantID).
Delete(&IMChannel{}).Error
if err := s.db.Where("agent_id = ? AND tenant_id = ? AND deleted_at IS NULL", agentID, tenantID).
Delete(&IMChannel{}).Error; err != nil {
return err
}
for i := range channels {
s.StopChannel(channels[i].ID)
s.publishChannelConfigChange(channels[i].ID)
}
return nil
}
// DeleteChannel soft-deletes a channel and stops it. Only deletes if the channel belongs to the given tenant.
func (s *Service) DeleteChannel(channelID string, tenantID uint64) error {
s.StopChannel(channelID)
result := s.db.Where("id = ? AND tenant_id = ?", channelID, tenantID).Delete(&IMChannel{})
if result.Error != nil {
return result.Error
@ -2807,6 +3047,8 @@ func (s *Service) DeleteChannel(channelID string, tenantID uint64) error {
if result.RowsAffected == 0 {
return fmt.Errorf("channel not found")
}
s.StopChannel(channelID)
s.publishChannelConfigChange(channelID)
return nil
}
@ -2827,6 +3069,7 @@ func (s *Service) ToggleChannel(channelID string, tenantID uint64) (*IMChannel,
} else {
s.StopChannel(channelID)
}
s.publishChannelConfigChange(channelID)
return &ch, nil
}

View File

@ -0,0 +1,336 @@
package im
import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/alicebob/miniredis/v2"
"github.com/gin-gonic/gin"
"github.com/redis/go-redis/v9"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
type lifecycleTestAdapter struct{}
func (*lifecycleTestAdapter) Platform() Platform { return Platform("test") }
func (*lifecycleTestAdapter) VerifyCallback(*gin.Context) error { return nil }
func (*lifecycleTestAdapter) ParseCallback(*gin.Context) (*IncomingMessage, error) {
return nil, nil
}
func (*lifecycleTestAdapter) SendReply(context.Context, *IncomingMessage, *ReplyMessage) error {
return nil
}
func (*lifecycleTestAdapter) HandleURLVerification(*gin.Context) bool { return false }
type lifecycleFactoryCounters struct {
starts atomic.Int32
stops atomic.Int32
}
func (c *lifecycleFactoryCounters) factory() AdapterFactory {
return func(context.Context, *IMChannel, func(context.Context, *IncomingMessage) error) (Adapter, context.CancelFunc, error) {
c.starts.Add(1)
var once sync.Once
return &lifecycleTestAdapter{}, func() {
once.Do(func() { c.stops.Add(1) })
}, nil
}
}
func newLifecycleTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:im-lifecycle-%d?mode=memory&cache=shared", time.Now().UnixNano())), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
// IMChannel's production schema uses PostgreSQL's uuid_generate_v4()
// default, which SQLite cannot parse. Keep an equivalent minimal table for
// lifecycle tests; IDs are assigned explicitly below.
if err := db.Exec(`CREATE TABLE im_channels (
id TEXT PRIMARY KEY,
tenant_id INTEGER NOT NULL,
agent_id TEXT NOT NULL,
platform TEXT NOT NULL,
name TEXT NOT NULL DEFAULT '',
enabled NUMERIC NOT NULL DEFAULT 1,
mode TEXT NOT NULL DEFAULT 'websocket',
output_mode TEXT NOT NULL DEFAULT 'stream',
knowledge_base_id TEXT DEFAULT '',
bot_identity TEXT NOT NULL DEFAULT '',
session_mode TEXT NOT NULL DEFAULT 'user',
credentials TEXT NOT NULL DEFAULT '{}',
created_at DATETIME,
updated_at DATETIME,
deleted_at DATETIME
)`).Error; err != nil {
t.Fatalf("create im_channels: %v", err)
}
return db
}
func newLifecycleTestService(db *gorm.DB, redisClient *redis.Client, instanceID string) *Service {
return &Service{
db: db,
channels: make(map[string]*channelState),
leaderRetries: make(map[string]*leaderRetryState),
adapterFactories: make(map[string]AdapterFactory),
redis: redisClient,
instanceID: instanceID,
stopCh: make(chan struct{}),
}
}
func createLifecycleChannel(t *testing.T, db *gorm.DB, id, agentID string) *IMChannel {
t.Helper()
channel := &IMChannel{
ID: id,
TenantID: 1,
AgentID: agentID,
Platform: "test",
Enabled: true,
Mode: "webhook",
OutputMode: "full",
SessionMode: string(SessionModeUser),
Credentials: types.JSON(`{"token":"v1"}`),
}
if err := db.Create(channel).Error; err != nil {
t.Fatalf("create channel: %v", err)
}
return channel
}
func TestEnsureChannelAdapterRefreshesStaleConfig(t *testing.T) {
db := newLifecycleTestDB(t)
channel := createLifecycleChannel(t, db, "channel-refresh", "agent-old")
counters := &lifecycleFactoryCounters{}
svc := newLifecycleTestService(db, nil, "instance-one")
svc.RegisterAdapterFactory("test", counters.factory())
t.Cleanup(svc.Stop)
if err := svc.StartChannel(channel); err != nil {
t.Fatalf("start initial channel: %v", err)
}
if err := db.Model(&IMChannel{}).Where("id = ?", channel.ID).
Updates(map[string]any{"agent_id": "agent-new", "credentials": types.JSON(`{"token":"v2"}`)}).Error; err != nil {
t.Fatalf("update durable channel: %v", err)
}
_, fresh, err := svc.EnsureChannelAdapter(channel.ID)
if err != nil {
t.Fatalf("ensure channel adapter: %v", err)
}
if fresh.AgentID != "agent-new" || string(fresh.Credentials) != `{"token":"v2"}` {
t.Fatalf("stale runtime config returned: agent=%q credentials=%s", fresh.AgentID, fresh.Credentials)
}
if counters.starts.Load() != 2 || counters.stops.Load() != 1 {
t.Fatalf("runtime was not rebuilt exactly once: starts=%d stops=%d", counters.starts.Load(), counters.stops.Load())
}
}
func TestEnsureChannelAdapterStopsDisabledCachedChannel(t *testing.T) {
db := newLifecycleTestDB(t)
channel := createLifecycleChannel(t, db, "channel-disabled", "agent")
counters := &lifecycleFactoryCounters{}
svc := newLifecycleTestService(db, nil, "instance-one")
svc.RegisterAdapterFactory("test", counters.factory())
t.Cleanup(svc.Stop)
if err := svc.StartChannel(channel); err != nil {
t.Fatalf("start channel: %v", err)
}
if err := db.Model(&IMChannel{}).Where("id = ?", channel.ID).Update("enabled", false).Error; err != nil {
t.Fatalf("disable channel: %v", err)
}
if _, _, err := svc.EnsureChannelAdapter(channel.ID); err == nil {
t.Fatal("EnsureChannelAdapter() expected disabled error")
}
if _, _, ok := svc.GetChannelAdapter(channel.ID); ok {
t.Fatal("disabled channel remained in runtime map")
}
if counters.stops.Load() != 1 {
t.Fatalf("cleanup calls = %d, want 1", counters.stops.Load())
}
}
func TestEnsureChannelAdapterKeepsRuntimeOnDatabaseFailure(t *testing.T) {
db := newLifecycleTestDB(t)
channel := createLifecycleChannel(t, db, "channel-db-failure", "agent")
counters := &lifecycleFactoryCounters{}
svc := newLifecycleTestService(db, nil, "instance-one")
svc.RegisterAdapterFactory("test", counters.factory())
t.Cleanup(svc.Stop)
if err := svc.StartChannel(channel); err != nil {
t.Fatalf("start channel: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("resolve sql DB: %v", err)
}
if err := sqlDB.Close(); err != nil {
t.Fatalf("close test DB: %v", err)
}
if _, _, err := svc.EnsureChannelAdapter(channel.ID); err == nil {
t.Fatal("EnsureChannelAdapter() expected database error")
}
if _, _, ok := svc.GetChannelAdapter(channel.ID); !ok {
t.Fatal("transient database failure tore down the cached runtime")
}
if counters.stops.Load() != 0 {
t.Fatalf("cleanup calls = %d, want 0 before service shutdown", counters.stops.Load())
}
}
func TestChannelConfigEventReloadsOtherReplica(t *testing.T) {
db := newLifecycleTestDB(t)
channel := createLifecycleChannel(t, db, "channel-pubsub", "agent-old")
redisServer := miniredis.RunT(t)
redisOne := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
redisTwo := redis.NewClient(&redis.Options{Addr: redisServer.Addr()})
t.Cleanup(func() { _ = redisOne.Close(); _ = redisTwo.Close() })
countersOne := &lifecycleFactoryCounters{}
countersTwo := &lifecycleFactoryCounters{}
svcOne := newLifecycleTestService(db, redisOne, "instance-one")
svcTwo := newLifecycleTestService(db, redisTwo, "instance-two")
svcOne.RegisterAdapterFactory("test", countersOne.factory())
svcTwo.RegisterAdapterFactory("test", countersTwo.factory())
svcOne.startChannelConfigSubscriber()
svcTwo.startChannelConfigSubscriber()
t.Cleanup(svcOne.Stop)
t.Cleanup(svcTwo.Stop)
if err := svcOne.StartChannel(channel); err != nil {
t.Fatalf("start channel on instance one: %v", err)
}
copyForTwo := *channel
if err := svcTwo.StartChannel(&copyForTwo); err != nil {
t.Fatalf("start channel on instance two: %v", err)
}
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
counts, err := redisOne.PubSubNumSub(context.Background(), RedisChannelConfig).Result()
if err == nil && counts[RedisChannelConfig] == 2 {
break
}
time.Sleep(10 * time.Millisecond)
}
updated := *channel
updated.AgentID = "agent-new"
updated.Credentials = types.JSON(`{"token":"v2"}`)
if err := svcOne.UpdateChannel(&updated); err != nil {
t.Fatalf("update channel: %v", err)
}
deadline = time.Now().Add(2 * time.Second)
reloaded := false
for time.Now().Before(deadline) {
_, runtimeChannel, ok := svcTwo.GetChannelAdapter(channel.ID)
if ok && runtimeChannel.AgentID == "agent-new" && string(runtimeChannel.Credentials) == `{"token":"v2"}` {
if countersTwo.starts.Load() < 2 || countersTwo.stops.Load() < 1 {
t.Fatalf("replica config changed without rebuilding runtime: starts=%d stops=%d", countersTwo.starts.Load(), countersTwo.stops.Load())
}
reloaded = true
break
}
time.Sleep(10 * time.Millisecond)
}
if !reloaded {
t.Fatal("second replica did not reload the published channel change")
}
if _, err := svcOne.ToggleChannel(channel.ID, channel.TenantID); err != nil {
t.Fatalf("disable channel: %v", err)
}
deadline = time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if _, _, ok := svcTwo.GetChannelAdapter(channel.ID); !ok {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("second replica did not stop the disabled channel")
}
func TestServiceStopIsIdempotent(t *testing.T) {
db := newLifecycleTestDB(t)
channel := createLifecycleChannel(t, db, "channel-stop", "agent")
counters := &lifecycleFactoryCounters{}
svc := newLifecycleTestService(db, nil, "instance-one")
svc.RegisterAdapterFactory("test", counters.factory())
if err := svc.StartChannel(channel); err != nil {
t.Fatalf("start channel: %v", err)
}
svc.Stop()
svc.Stop()
if counters.stops.Load() != 1 {
t.Fatalf("cleanup calls = %d, want exactly 1", counters.stops.Load())
}
if err := svc.StartChannel(channel); err == nil {
t.Fatal("StartChannel() succeeded after service shutdown")
}
}
func TestSameChannelRuntimeConfigUsesSemanticCredentials(t *testing.T) {
now := time.Now()
cached := &IMChannel{
ID: "channel",
TenantID: 1,
AgentID: "agent",
Platform: "test",
Enabled: true,
Mode: "webhook",
OutputMode: "full",
SessionMode: string(SessionModeUser),
Credentials: types.JSON(`{"token":"secret","timeout":10}`),
UpdatedAt: now,
}
fresh := *cached
fresh.Credentials = types.JSON(`{"timeout":10,"token":"secret"}`)
fresh.UpdatedAt = now.Truncate(time.Microsecond)
if !sameChannelRuntimeConfig(cached, &fresh) {
t.Fatal("semantic-equivalent credentials or timestamp precision triggered a rebuild")
}
fresh.Credentials = types.JSON(`{"timeout":10,"token":"changed"}`)
if sameChannelRuntimeConfig(cached, &fresh) {
t.Fatal("changed credentials did not trigger a rebuild")
}
}
func TestLeaderElectionFailsClosedWhenRedisUnavailable(t *testing.T) {
db := newLifecycleTestDB(t)
channel := createLifecycleChannel(t, db, "channel-redis-down", "agent")
channel.Mode = "websocket"
counters := &lifecycleFactoryCounters{}
redisClient := redis.NewClient(&redis.Options{Addr: "127.0.0.1:1"})
_ = redisClient.Close()
svc := newLifecycleTestService(db, redisClient, "instance-one")
svc.RegisterAdapterFactory("test", counters.factory())
t.Cleanup(svc.Stop)
if err := svc.StartChannel(channel); err != nil {
t.Fatalf("StartChannel() should schedule a retry, got %v", err)
}
if err := svc.StartChannel(channel); err != nil {
t.Fatalf("second StartChannel() should replace the retry, got %v", err)
}
if counters.starts.Load() != 0 {
t.Fatalf("factory starts = %d, want 0 while leader election is unavailable", counters.starts.Load())
}
svc.mu.RLock()
retryCount := len(svc.leaderRetries)
svc.mu.RUnlock()
if retryCount != 1 {
t.Fatalf("leader retry goroutines = %d, want one per channel", retryCount)
}
}

View File

@ -63,7 +63,14 @@ func NewFactory() im.AdapterFactory {
}()
adapter := NewWSAdapter(client)
return adapter, wsCancel, nil
// Cancelling the context alone does not unblock gorilla/websocket's
// ReadMessage. Close the socket first so disabling, reconfiguring, or
// shutting down a channel stops message delivery synchronously.
stop := func() {
client.Stop()
wsCancel()
}
return adapter, stop, nil
default:
return nil, nil, fmt.Errorf("unknown WeCom mode: %s", mode)