62 lines
2.2 KiB
Python
62 lines
2.2 KiB
Python
"""Celery 应用:队列划分、并发策略与周期调度(派发器 + 断点续跑)。
|
||
|
||
启动方式(详见 README):
|
||
celery -A app.worker worker -Q generate,audio,qc,compose,io,default -c 2
|
||
celery -A app.worker beat
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from celery import Celery
|
||
from celery.schedules import crontab # noqa: F401 保留给自定义周期表达式
|
||
|
||
from app.config import settings
|
||
|
||
# 队列划分:不同工种的资源画像不同(GPU/网络受限的生成 vs CPU 密集的合成)
|
||
QUEUES = {"generate_shot": "generate", "voiceover": "audio", "qc_shot": "qc",
|
||
"compose": "compose", "import": "io", "asset": "io", "platform_asset": "io"}
|
||
|
||
TASK_ROUTES = {
|
||
"app.jobs.generate_shot": {"queue": "generate"},
|
||
"app.jobs.voiceover": {"queue": "audio"},
|
||
"app.jobs.qc_shot": {"queue": "qc"},
|
||
"app.jobs.compose": {"queue": "compose"},
|
||
"app.jobs.import_package": {"queue": "io"},
|
||
"app.jobs.asset": {"queue": "io"},
|
||
"app.jobs.platform_asset": {"queue": "io"},
|
||
}
|
||
|
||
celery_app = Celery("nex_animation", broker=settings.broker_url, backend=settings.result_backend)
|
||
celery_app.conf.update(
|
||
task_default_queue="default",
|
||
task_routes=TASK_ROUTES,
|
||
task_track_started=True,
|
||
task_acks_late=True,
|
||
task_reject_on_worker_lost=True,
|
||
worker_prefetch_multiplier=1, # 长耗时任务:一次只取一条,避免堆积
|
||
worker_max_tasks_per_child=50,
|
||
task_serializer="json",
|
||
result_serializer="json",
|
||
accept_content=["json"],
|
||
timezone="Asia/Shanghai",
|
||
enable_utc=True,
|
||
broker_connection_retry_on_startup=True,
|
||
result_expires=60 * 60 * 24,
|
||
task_time_limit=6 * 3600,
|
||
task_soft_time_limit=5 * 3600,
|
||
beat_schedule={
|
||
"dispatch-due": {
|
||
"task": "app.jobs.dispatch_due",
|
||
"schedule": max(1, settings.dispatch_interval_seconds),
|
||
"options": {"queue": "default", "expires": max(1, settings.dispatch_interval_seconds - 1)},
|
||
},
|
||
"resume-interrupted": {
|
||
"task": "app.jobs.resume_interrupted",
|
||
"schedule": 60.0,
|
||
"options": {"queue": "default"},
|
||
},
|
||
},
|
||
)
|
||
|
||
# 任务注册放在最后导入,避免循环依赖
|
||
from app.jobs import tasks # noqa: E402,F401
|