nex_animation/backend/app/worker.py

62 lines
2.2 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Celery 应用:队列划分、并发策略与周期调度(派发器 + 断点续跑)。
启动方式(详见 README):
celery -A app.worker worker -Q generate,audio,qc,compose,io,default -c 2
celery -A app.worker beat
"""
from __future__ import annotations
from celery import Celery
from celery.schedules import crontab # noqa: F401 保留给自定义周期表达式
from app.config import settings
# 队列划分:不同工种的资源画像不同(GPU/网络受限的生成 vs CPU 密集的合成)
QUEUES = {"generate_shot": "generate", "voiceover": "audio", "qc_shot": "qc",
"compose": "compose", "import": "io", "asset": "io", "platform_asset": "io"}
TASK_ROUTES = {
"app.jobs.generate_shot": {"queue": "generate"},
"app.jobs.voiceover": {"queue": "audio"},
"app.jobs.qc_shot": {"queue": "qc"},
"app.jobs.compose": {"queue": "compose"},
"app.jobs.import_package": {"queue": "io"},
"app.jobs.asset": {"queue": "io"},
"app.jobs.platform_asset": {"queue": "io"},
}
celery_app = Celery("nex_animation", broker=settings.broker_url, backend=settings.result_backend)
celery_app.conf.update(
task_default_queue="default",
task_routes=TASK_ROUTES,
task_track_started=True,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_prefetch_multiplier=1, # 长耗时任务:一次只取一条,避免堆积
worker_max_tasks_per_child=50,
task_serializer="json",
result_serializer="json",
accept_content=["json"],
timezone="Asia/Shanghai",
enable_utc=True,
broker_connection_retry_on_startup=True,
result_expires=60 * 60 * 24,
task_time_limit=6 * 3600,
task_soft_time_limit=5 * 3600,
beat_schedule={
"dispatch-due": {
"task": "app.jobs.dispatch_due",
"schedule": max(1, settings.dispatch_interval_seconds),
"options": {"queue": "default", "expires": max(1, settings.dispatch_interval_seconds - 1)},
},
"resume-interrupted": {
"task": "app.jobs.resume_interrupted",
"schedule": 60.0,
"options": {"queue": "default"},
},
},
)
# 任务注册放在最后导入,避免循环依赖
from app.jobs import tasks # noqa: E402,F401