fix(server): run ResourceCleaner on signal shutdown instead of racing Fatalf exit (#3804)
* fix(server): signal shutdown raced Fatalf exit before ResourceCleaner ran
The signal goroutine closed the listener before calling server.Shutdown,
so Serve returned the raw accept error ('use of closed network connection')
instead of ErrServerClosed. The invoke error path then hit logger.Fatalf
(os.Exit(1)) in the same millisecond, killing the process before the
shutdown goroutine reached resourceCleaner.Cleanup. Every registered
cleaner — BrowserSkill daemon, sandbox reapers, IMService, AntsPool,
Langfuse — was skipped on every SIGTERM/SIGINT, leaking the shared bsk
daemon process and its temp home on each restart.
Two changes:
- Drop the manual listener.Close(): Shutdown closes all listeners as its
first step, releasing the port just as fast without racing Serve.
- Give resourceCleaner.Cleanup its own fresh timeout context so a slow
connection drain cannot leave Cleanup with an already-expired context,
which made it skip every entry without running any.
* fix(server): keep signal cleanup inside one grace period
Desktop quit still closed the listener before Shutdown and returned
before ResourceCleaner finished, so Fatalf could skip every hook.
Share ShutdownTimeout between drain and cleanup, run BrowserSkill
first, and do not drop later hooks when the deadline expires.
* fix(server): satisfy errcheck and line-length lint
---------
Co-authored-by: vicfei <vicfei@users.noreply.github.com>
Co-authored-by: wizardchen <wizardchen@tencent.com>
main
parent
c819abd958
commit
ec0ca46334
|
|
@ -17,13 +17,15 @@ type App struct {
|
|||
apiLanBaseURL string
|
||||
listenPublic bool
|
||||
shutdownCh chan struct{}
|
||||
shutdownDone chan struct{}
|
||||
pickingDir atomic.Bool
|
||||
}
|
||||
|
||||
// NewApp creates a new App application struct.
|
||||
func NewApp() *App {
|
||||
return &App{
|
||||
shutdownCh: make(chan struct{}, 1),
|
||||
shutdownCh: make(chan struct{}, 1),
|
||||
shutdownDone: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -33,7 +35,11 @@ func (a *App) startup(ctx context.Context) {
|
|||
}
|
||||
|
||||
func (a *App) shutdown(ctx context.Context) {
|
||||
a.shutdownCh <- struct{}{}
|
||||
select {
|
||||
case a.shutdownCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-a.shutdownDone
|
||||
}
|
||||
|
||||
// GetAPIBaseURL returns the local HTTP base URL for REST API calls (e.g. http://127.0.0.1:PORT/api/v1).
|
||||
|
|
|
|||
|
|
@ -268,6 +268,7 @@ func main() {
|
|||
|
||||
// Run backend in a separate goroutine
|
||||
go func() {
|
||||
defer close(app.shutdownDone)
|
||||
err := c.Invoke(func(
|
||||
cfg *config.Config,
|
||||
router *gin.Engine,
|
||||
|
|
@ -297,23 +298,28 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
// Handle graceful shutdown from Wails OnShutdown hook
|
||||
// Wails OnShutdown returns only after this goroutine closes
|
||||
// serverStopped. Closing the listener by hand makes Serve return
|
||||
// "use of closed network connection" instead of ErrServerClosed,
|
||||
// and the error path calls logger.Fatalf → os.Exit(1) before cleanup.
|
||||
serverStopped := make(chan struct{})
|
||||
go func() {
|
||||
defer close(serverStopped)
|
||||
<-app.shutdownCh
|
||||
logger.Infof(context.Background(), "Wails shutting down, stopping Go backend...")
|
||||
|
||||
listener.Close()
|
||||
shutdownTimeout := cfg.Server.ShutdownTimeout
|
||||
if shutdownTimeout == 0 {
|
||||
shutdownTimeout = 30 * time.Second
|
||||
}
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
drainBudget, cleanupBudget := runtime.ShutdownBudgets(cfg.Server.ShutdownTimeout)
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), drainBudget)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
server.Close()
|
||||
}
|
||||
resourceCleaner.Cleanup(shutdownCtx)
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), cleanupBudget)
|
||||
defer cleanupCancel()
|
||||
if errs := resourceCleaner.Cleanup(cleanupCtx); len(errs) > 0 {
|
||||
logger.Errorf(context.Background(), "Errors occurred during resource cleanup: %v", errs)
|
||||
}
|
||||
}()
|
||||
|
||||
// Also listen for OS signals just in case
|
||||
|
|
@ -321,13 +327,17 @@ func main() {
|
|||
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-signals
|
||||
app.shutdownCh <- struct{}{} // trigger shutdown
|
||||
select {
|
||||
case app.shutdownCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}()
|
||||
|
||||
logger.Infof(context.Background(), "Server is running at %s (proxy -> %s)", tcpAddr.String(), app.backendURL)
|
||||
if err := server.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
return fmt.Errorf("server error: %v", err)
|
||||
}
|
||||
<-serverStopped
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -99,31 +99,34 @@ func main() {
|
|||
sig := <-signals
|
||||
logger.Infof(context.Background(), "Received signal: %v, starting server shutdown...", sig)
|
||||
|
||||
// Close listener first to release port immediately,
|
||||
// so the next process can bind during our graceful drain.
|
||||
listener.Close()
|
||||
|
||||
shutdownTimeout := cfg.Server.ShutdownTimeout
|
||||
if shutdownTimeout == 0 {
|
||||
shutdownTimeout = 30 * time.Second
|
||||
}
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
drainBudget, cleanupBudget := runtime.ShutdownBudgets(cfg.Server.ShutdownTimeout)
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), drainBudget)
|
||||
defer shutdownCancel()
|
||||
|
||||
// Second signal → force close all connections immediately
|
||||
// Second signal → force close all connections immediately.
|
||||
go func() {
|
||||
sig := <-signals
|
||||
logger.Warnf(context.Background(), "Received second signal: %v, forcing shutdown...", sig)
|
||||
server.Close()
|
||||
}()
|
||||
|
||||
// Do NOT close the listener manually before Shutdown. Serve would
|
||||
// return the raw accept error ("use of closed network connection")
|
||||
// instead of ErrServerClosed, and logger.Fatalf → os.Exit(1) would
|
||||
// kill the process before ResourceCleaner runs. Shutdown closes
|
||||
// listeners first, so the port is released before the drain.
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
logger.Errorf(context.Background(), "Server forced to shutdown: %v", err)
|
||||
server.Close()
|
||||
}
|
||||
|
||||
logger.Info(context.Background(), "Cleaning up resources...")
|
||||
errs := resourceCleaner.Cleanup(shutdownCtx)
|
||||
// Fresh context sized from the reserved slice of ShutdownTimeout.
|
||||
// Drain must not hand cleanup an already-expired context, and the
|
||||
// two slices must still add up to one supervisor grace period.
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), cleanupBudget)
|
||||
defer cleanupCancel()
|
||||
errs := resourceCleaner.Cleanup(cleanupCtx)
|
||||
if len(errs) > 0 {
|
||||
logger.Errorf(context.Background(), "Errors occurred during resource cleanup: %v", errs)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func startTestServer(t *testing.T) (*http.Server, net.Listener, <-chan error) {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
srv := &http.Server{Handler: http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})}
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- srv.Serve(ln)
|
||||
}()
|
||||
waitUntilAccepting(t, ln.Addr().String())
|
||||
return srv, ln, errCh
|
||||
}
|
||||
|
||||
func waitUntilAccepting(t *testing.T, addr string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", addr, 50*time.Millisecond)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("server at %s did not accept", addr)
|
||||
}
|
||||
|
||||
func TestShutdownReturnsErrServerClosed(t *testing.T) {
|
||||
srv, _, errCh := startTestServer(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(ctx); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
t.Fatalf("Serve err = %v, want ErrServerClosed", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Serve did not return")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualListenerCloseIsNotErrServerClosed(t *testing.T) {
|
||||
_, ln, errCh := startTestServer(t)
|
||||
if err := ln.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err == nil || errors.Is(err, http.ErrServerClosed) {
|
||||
t.Fatalf("Serve err = %v, want the raw accept error", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Serve did not return")
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
|
|
@ -102,6 +103,13 @@ func (r *AuditLogRetentionRunner) Start(ctx context.Context) {
|
|||
// If Start was never called, Stop returns immediately (no doneCh to
|
||||
// wait on — see the `started` flag in the struct comment).
|
||||
func (r *AuditLogRetentionRunner) Stop() {
|
||||
r.StopWithin(0)
|
||||
}
|
||||
|
||||
// StopWithin is Stop with a bound. timeout <= 0 waits until the loop
|
||||
// returns. A positive timeout continues shutdown while a purge is still
|
||||
// inside its own database deadline, instead of holding every later hook.
|
||||
func (r *AuditLogRetentionRunner) StopWithin(timeout time.Duration) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -111,7 +119,10 @@ func (r *AuditLogRetentionRunner) Stop() {
|
|||
r.stopOnce.Do(func() {
|
||||
close(r.stopCh)
|
||||
})
|
||||
<-r.doneCh
|
||||
if runtime.WaitFor(r.doneCh, timeout) {
|
||||
return
|
||||
}
|
||||
logger.Warnf(context.Background(), "[audit-retention] sweep still running after %s; continuing shutdown", timeout)
|
||||
}
|
||||
|
||||
// loop runs the actual sweep cadence. Uses a fresh context.Background
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
"github.com/Tencent/WeKnora/internal/config"
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/robfig/cron/v3"
|
||||
|
|
@ -119,13 +120,20 @@ func (h *HousekeepingService) Start(ctx context.Context) error {
|
|||
|
||||
// Stop halts the cron and waits for in-flight sweeps to finish.
|
||||
func (h *HousekeepingService) Stop() {
|
||||
h.StopWithin(0)
|
||||
}
|
||||
|
||||
// StopWithin is Stop with a bound. timeout <= 0 waits for the in-flight sweep.
|
||||
func (h *HousekeepingService) StopWithin(timeout time.Duration) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !h.started {
|
||||
return
|
||||
}
|
||||
c := h.cron.Stop()
|
||||
<-c.Done()
|
||||
if !runtime.WaitFor(h.cron.Stop().Done(), timeout) {
|
||||
logger.Warnf(context.Background(),
|
||||
"[Housekeeping] in-flight sweep still running after %s; continuing shutdown", timeout)
|
||||
}
|
||||
h.started = false
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/sandbox"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -874,12 +875,19 @@ func (s *TenantSkillService) Start(ctx context.Context) error {
|
|||
|
||||
// Stop halts the cron and waits for in-flight sweeps to finish.
|
||||
func (s *TenantSkillService) Stop() {
|
||||
s.StopWithin(0)
|
||||
}
|
||||
|
||||
// StopWithin is Stop with a bound. timeout <= 0 waits for the in-flight sweep.
|
||||
func (s *TenantSkillService) StopWithin(timeout time.Duration) {
|
||||
s.cronMu.Lock()
|
||||
defer s.cronMu.Unlock()
|
||||
if !s.started {
|
||||
return
|
||||
}
|
||||
c := s.cron.Stop()
|
||||
<-c.Done()
|
||||
if !runtime.WaitFor(s.cron.Stop().Done(), timeout) {
|
||||
logger.Warnf(context.Background(),
|
||||
"[skill] in-flight reaper still running after %s; continuing shutdown", timeout)
|
||||
}
|
||||
s.started = false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,38 +4,43 @@ import (
|
|||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// cleanupStepTimeout bounds one shutdown hook that waits on in-flight work
|
||||
// (cron jobs, retention sweeps). Several of those hooks run before child
|
||||
// processes are reaped; an unbounded wait used to hold SIGTERM until the
|
||||
// supervisor killed the process and skipped the rest.
|
||||
const cleanupStepTimeout = 5 * time.Second
|
||||
|
||||
type cleanupEntry struct {
|
||||
name string
|
||||
fn types.CleanupFunc
|
||||
}
|
||||
|
||||
// ResourceCleaner is a resource cleaner that can be used to clean up resources
|
||||
type ResourceCleaner struct {
|
||||
mu sync.Mutex
|
||||
cleanups []types.CleanupFunc
|
||||
cleanups []cleanupEntry
|
||||
}
|
||||
|
||||
// NewResourceCleaner creates a new resource cleaner
|
||||
func NewResourceCleaner() interfaces.ResourceCleaner {
|
||||
return &ResourceCleaner{
|
||||
cleanups: make([]types.CleanupFunc, 0),
|
||||
cleanups: make([]cleanupEntry, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Register registers a cleanup function
|
||||
// Note: the cleanup function will be executed in reverse order (the last registered will be executed first)
|
||||
// Register registers a cleanup function.
|
||||
// The cleanup function runs in reverse order (the last registered runs first).
|
||||
func (c *ResourceCleaner) Register(cleanup types.CleanupFunc) {
|
||||
if cleanup == nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.cleanups = append(c.cleanups, cleanup)
|
||||
c.register("", cleanup)
|
||||
}
|
||||
|
||||
// RegisterWithName registers a cleanup function with a name, for logging tracking
|
||||
// RegisterWithName registers a cleanup function with a name, for logging tracking.
|
||||
func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFunc) {
|
||||
if cleanup == nil {
|
||||
return
|
||||
|
|
@ -52,28 +57,64 @@ func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFun
|
|||
return err
|
||||
}
|
||||
|
||||
c.Register(wrappedCleanup)
|
||||
c.register(name, wrappedCleanup)
|
||||
}
|
||||
|
||||
// Cleanup executes all cleanup functions
|
||||
// Even if a cleanup function fails, other cleanup functions will still be executed
|
||||
func (c *ResourceCleaner) register(name string, cleanup types.CleanupFunc) {
|
||||
if cleanup == nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.cleanups = append(c.cleanups, cleanupEntry{name: name, fn: cleanup})
|
||||
}
|
||||
|
||||
// Promote moves every callback registered under name to the end of the list.
|
||||
// Cleanup runs in reverse, so the promoted callbacks run first. BrowserSkill
|
||||
// is registered during early container wiring and would otherwise run last,
|
||||
// after hooks that wait on in-flight jobs.
|
||||
func (c *ResourceCleaner) Promote(name string) {
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
matched := make([]cleanupEntry, 0, 1)
|
||||
rest := make([]cleanupEntry, 0, len(c.cleanups))
|
||||
for _, entry := range c.cleanups {
|
||||
if entry.name == name {
|
||||
matched = append(matched, entry)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, entry)
|
||||
}
|
||||
c.cleanups = append(rest, matched...)
|
||||
}
|
||||
|
||||
// Cleanup executes all cleanup functions.
|
||||
// A cancelled or expired context does not skip remaining functions: the
|
||||
// deadline used to return on the first select and drop every later hook,
|
||||
// including ones registered earliest (BrowserSkill). Context errors are not
|
||||
// reported as hook failures when the hooks themselves still ran.
|
||||
func (c *ResourceCleaner) Cleanup(ctx context.Context) (errs []error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Execute cleanup functions in reverse order (the last registered will be executed first)
|
||||
for i := len(c.cleanups) - 1; i >= 0; i-- {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errs = append(errs, ctx.Err())
|
||||
return errs
|
||||
default:
|
||||
if err := c.cleanups[i](); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
for i := len(c.cleanups) - 1; i >= 0; i-- {
|
||||
if err := c.cleanups[i].fn(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
log.Printf("Resource cleanup continued past its deadline: %v", err)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
|
|
@ -82,5 +123,5 @@ func (c *ResourceCleaner) Reset() {
|
|||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.cleanups = make([]types.CleanupFunc, 0)
|
||||
c.cleanups = make([]cleanupEntry, 0)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCleanupRunsRemainingHooksAfterDeadline(t *testing.T) {
|
||||
cleaner := NewResourceCleaner()
|
||||
var ran []string
|
||||
// Registered first, so it runs last unless promoted. This is BrowserSkill's
|
||||
// position: an earlier slow hook must not cause it to be skipped.
|
||||
cleaner.RegisterWithName("tail", func() error {
|
||||
ran = append(ran, "tail")
|
||||
return nil
|
||||
})
|
||||
cleaner.RegisterWithName("slow", func() error {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
ran = append(ran, "slow")
|
||||
return errors.New("slow failed")
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
errs := cleaner.Cleanup(ctx)
|
||||
if len(ran) != 2 || ran[0] != "slow" || ran[1] != "tail" {
|
||||
t.Fatalf("ran = %v, want slow then tail", ran)
|
||||
}
|
||||
if len(errs) != 1 || errs[0].Error() != "slow failed" {
|
||||
t.Fatalf("errs = %v, want only the hook error", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromoteRunsNamedHookFirst(t *testing.T) {
|
||||
cleaner := NewResourceCleaner()
|
||||
var ran []string
|
||||
cleaner.RegisterWithName("BrowserSkill", func() error {
|
||||
ran = append(ran, "browser")
|
||||
return nil
|
||||
})
|
||||
cleaner.RegisterWithName("later", func() error {
|
||||
ran = append(ran, "later")
|
||||
return nil
|
||||
})
|
||||
cleaner.Promote("BrowserSkill")
|
||||
|
||||
if errs := cleaner.Cleanup(context.Background()); len(errs) != 0 {
|
||||
t.Fatalf("errs = %v", errs)
|
||||
}
|
||||
if len(ran) != 2 || ran[0] != "browser" || ran[1] != "later" {
|
||||
t.Fatalf("ran = %v, want browser then later", ran)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupAlreadyCanceledContextStillRunsHooks(t *testing.T) {
|
||||
cleaner := NewResourceCleaner()
|
||||
var n int
|
||||
cleaner.Register(func() error {
|
||||
n++
|
||||
return nil
|
||||
})
|
||||
cleaner.Register(func() error {
|
||||
n++
|
||||
return nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if errs := cleaner.Cleanup(ctx); len(errs) != 0 {
|
||||
t.Fatalf("errs = %v", errs)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("ran %d hooks, want 2", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -615,6 +615,14 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
// them only after the matching handlers are ready.
|
||||
must(container.Invoke(recoverPendingWikiTasks))
|
||||
|
||||
// BrowserSkill is registered when its manager is constructed, which is
|
||||
// early, so reverse-order cleanup would run it last. Force the manager
|
||||
// to exist, then run that hook first so a slow cron stop cannot leave
|
||||
// the daemon alive until the process is killed.
|
||||
must(container.Invoke(func(cleaner interfaces.ResourceCleaner, _ *browserskill.Manager) {
|
||||
cleaner.Promote("BrowserSkill")
|
||||
}))
|
||||
|
||||
logger.Infof(ctx, "[Container] Container initialization completed successfully")
|
||||
return container
|
||||
}
|
||||
|
|
@ -1911,7 +1919,7 @@ func startDataSourceScheduler(scheduler *datasource.Scheduler, cleaner interface
|
|||
}
|
||||
|
||||
cleaner.RegisterWithName("DataSourceScheduler", func() error {
|
||||
scheduler.Stop()
|
||||
scheduler.StopWithin(cleanupStepTimeout)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
@ -1929,7 +1937,7 @@ func startHousekeepingService(svc *service.HousekeepingService, cleaner interfac
|
|||
logger.Warnf(context.Background(), "[Container] housekeeping start failed: %v", err)
|
||||
}
|
||||
cleaner.RegisterWithName("KnowledgeHousekeeping", func() error {
|
||||
svc.Stop()
|
||||
svc.StopWithin(cleanupStepTimeout)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
@ -1945,7 +1953,7 @@ func startTenantSkillReaper(svc *service.TenantSkillService, cleaner interfaces.
|
|||
logger.Warnf(context.Background(), "[Container] tenant skill reaper start failed: %v", err)
|
||||
}
|
||||
cleaner.RegisterWithName("TenantSkillReaper", func() error {
|
||||
svc.Stop()
|
||||
svc.StopWithin(cleanupStepTimeout)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
@ -2021,7 +2029,7 @@ func startAuditLogRetention(
|
|||
) {
|
||||
runner.Start(context.Background())
|
||||
cleaner.RegisterWithName("AuditLogRetentionRunner", func() error {
|
||||
runner.Stop()
|
||||
runner.StopWithin(cleanupStepTimeout)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/runtime"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
|
|
@ -92,8 +93,21 @@ func (s *Scheduler) Start(ctx context.Context) error {
|
|||
|
||||
// Stop gracefully stops the cron runner and waits for running jobs to finish.
|
||||
func (s *Scheduler) Stop() {
|
||||
ctx := s.cron.Stop()
|
||||
<-ctx.Done()
|
||||
s.StopWithin(0)
|
||||
}
|
||||
|
||||
// StopWithin is Stop with a bound. timeout <= 0 waits until in-flight jobs
|
||||
// finish. A positive timeout lets shutdown continue so one long sync cannot
|
||||
// hold every later cleanup hook, including child-process reaping.
|
||||
func (s *Scheduler) StopWithin(timeout time.Duration) {
|
||||
if s == nil || s.cron == nil {
|
||||
return
|
||||
}
|
||||
if runtime.WaitFor(s.cron.Stop().Done(), timeout) {
|
||||
return
|
||||
}
|
||||
logger.Warnf(context.Background(),
|
||||
"[Scheduler] in-flight sync still running after %s; continuing shutdown", timeout)
|
||||
}
|
||||
|
||||
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
package runtime
|
||||
|
||||
import "time"
|
||||
|
||||
// DefaultShutdownTimeout is used when server.shutdown_timeout is unset.
|
||||
const DefaultShutdownTimeout = 30 * time.Second
|
||||
|
||||
// CleanupBudget is reserved from the configured shutdown timeout so connection
|
||||
// drain cannot consume the whole grace period before process cleanup runs.
|
||||
// Supervisors often use the same number for termination grace (Kubernetes
|
||||
// defaults to 30s), so drain and cleanup share one budget instead of stacking.
|
||||
const CleanupBudget = 5 * time.Second
|
||||
|
||||
// ShutdownBudgets splits total into an HTTP drain budget and a cleanup budget.
|
||||
// Their sum equals total. total <= 0 uses DefaultShutdownTimeout. When total
|
||||
// is too small to reserve CleanupBudget, the two halves are equal.
|
||||
func ShutdownBudgets(total time.Duration) (drain, cleanup time.Duration) {
|
||||
if total <= 0 {
|
||||
total = DefaultShutdownTimeout
|
||||
}
|
||||
cleanup = CleanupBudget
|
||||
if cleanup >= total {
|
||||
cleanup = total / 2
|
||||
if cleanup <= 0 {
|
||||
cleanup = total
|
||||
}
|
||||
}
|
||||
drain = total - cleanup
|
||||
if drain <= 0 {
|
||||
drain = total
|
||||
}
|
||||
return drain, cleanup
|
||||
}
|
||||
|
||||
// WaitFor reports whether ch was received before timeout. timeout <= 0 waits
|
||||
// until ch is received. The boolean is false only when the timeout wins.
|
||||
func WaitFor(ch <-chan struct{}, timeout time.Duration) bool {
|
||||
if timeout <= 0 {
|
||||
<-ch
|
||||
return true
|
||||
}
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
case <-timer.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
package runtime
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestShutdownBudgetsShareOneDeadline(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
total time.Duration
|
||||
wantDrain time.Duration
|
||||
wantCleanup time.Duration
|
||||
}{
|
||||
{name: "unset", total: 0, wantDrain: 25 * time.Second, wantCleanup: 5 * time.Second},
|
||||
{name: "default", total: 30 * time.Second, wantDrain: 25 * time.Second, wantCleanup: 5 * time.Second},
|
||||
{name: "fits reserve", total: 10 * time.Second, wantDrain: 5 * time.Second, wantCleanup: 5 * time.Second},
|
||||
{name: "split small", total: 4 * time.Second, wantDrain: 2 * time.Second, wantCleanup: 2 * time.Second},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
drain, cleanup := ShutdownBudgets(tt.total)
|
||||
if drain != tt.wantDrain || cleanup != tt.wantCleanup {
|
||||
t.Fatalf("ShutdownBudgets(%s) = %s drain + %s cleanup, want %s + %s",
|
||||
tt.total, drain, cleanup, tt.wantDrain, tt.wantCleanup)
|
||||
}
|
||||
total := tt.total
|
||||
if total <= 0 {
|
||||
total = DefaultShutdownTimeout
|
||||
}
|
||||
if drain+cleanup != total {
|
||||
t.Fatalf("budgets sum to %s, want %s", drain+cleanup, total)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitFor(t *testing.T) {
|
||||
closed := make(chan struct{})
|
||||
close(closed)
|
||||
if !WaitFor(closed, time.Millisecond) {
|
||||
t.Fatal("closed channel should be received")
|
||||
}
|
||||
|
||||
never := make(chan struct{})
|
||||
start := time.Now()
|
||||
if WaitFor(never, 20*time.Millisecond) {
|
||||
t.Fatal("timeout should lose")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 500*time.Millisecond {
|
||||
t.Fatalf("timeout waited %s", elapsed)
|
||||
}
|
||||
|
||||
later := make(chan struct{})
|
||||
go func() {
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
close(later)
|
||||
}()
|
||||
if !WaitFor(later, 0) {
|
||||
t.Fatal("non-positive timeout should wait for the channel")
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,9 @@ import (
|
|||
type ResourceCleaner interface {
|
||||
Register(cleanup types.CleanupFunc)
|
||||
RegisterWithName(name string, cleanup types.CleanupFunc)
|
||||
// Promote moves callbacks registered under name so Cleanup runs them
|
||||
// first. Cleanup's order is reverse registration order.
|
||||
Promote(name string)
|
||||
Cleanup(ctx context.Context) []error
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ flowchart LR
|
|||
| `server.port` | int | 8080 | HTTP 监听端口,校验范围 1–65535 |
|
||||
| `server.host` | string | "0.0.0.0" | 监听地址 |
|
||||
| `server.log_path` | string | 空 | 日志文件路径(也可用环境变量 `LOG_PATH`) |
|
||||
| `server.shutdown_timeout` | duration | 30s | 优雅停机超时 |
|
||||
| `server.shutdown_timeout` | duration | 30s | 优雅停机总预算。连接排空与资源清理共用,默认留 5s 给清理 |
|
||||
|
||||
### conversation(`ConversationConfig`)——检索问答管线
|
||||
|
||||
|
|
|
|||
|
|
@ -119,10 +119,10 @@ flowchart TD
|
|||
H --> I["systemSettingSvc.SubscribeRedis(ctx)<br/>订阅 system_settings 变更 (Lite 模式 no-op)"]
|
||||
I --> J["signal.Notify(shutdownSignals) + server.Serve(listener)"]
|
||||
J --> K{"收到第一个信号?"}
|
||||
K -->|是| L["listener.Close() 立即释放端口<br/>server.Shutdown(ctx, ShutdownTimeout 默认 30s) 优雅排空"]
|
||||
K -->|是| L["server.Shutdown 先关 listener 再排空<br/>排空预算 = ShutdownTimeout − 5s"]
|
||||
L --> M{"排空期间收到第二个信号?"}
|
||||
M -->|是| N["server.Close() 强制断开所有连接"]
|
||||
M -->|否| O["resourceCleaner.Cleanup(ctx)<br/>ants 池 / Langfuse / 调度器逐个析构"]
|
||||
M -->|否| O["resourceCleaner.Cleanup<br/>BrowserSkill 最先执行;到期不跳过后续钩子"]
|
||||
N --> O
|
||||
O --> P["进程退出"]
|
||||
```
|
||||
|
|
@ -130,7 +130,7 @@ flowchart TD
|
|||
要点:
|
||||
|
||||
- **引导失败不阻断启动**:`bootstrap.go` 对失败记录 `logger.Warnf`。系统管理员引导仅在部署尚无系统管理员时生效,重启不会恢复已撤销的权限;
|
||||
- **两段式优雅退出**:第一个 SIGTERM/SIGINT 先关 listener(新进程可立即绑定端口)再 `Shutdown` 排空存量连接;第二个信号强制 `Close`;
|
||||
- **两段式优雅退出**:第一个 SIGTERM/SIGINT 调用 `server.Shutdown`(它会先关闭 listener,端口随即可被新进程绑定),在 `ShutdownTimeout` 里为资源清理预留 5s,其余时间排空存量连接。不要在 `Shutdown` 之前手动 `listener.Close()`:`Serve` 会返回 `use of closed network connection` 而不是 `ErrServerClosed`,`logger.Fatalf` 会在清理前把进程杀掉。第二个信号强制 `Close`;
|
||||
- **端口占用重试**:`listenWithRetry` 以 300ms 起步指数退避重试 10 次(滚动重启场景旧进程尚未释放端口时避免直接失败)。
|
||||
|
||||
## 路由组织与 RBAC 装配(internal/router) {#_4-路由组织与-rbac-装配-internal-router}
|
||||
|
|
|
|||
Loading…
Reference in New Issue