45 lines
1.7 KiB
Python
45 lines
1.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
|
|
from celery import Celery
|
|
from celery.schedules import crontab
|
|
from kombu import Exchange, Queue
|
|
from maxkb import settings
|
|
from .heartbeat import *
|
|
|
|
# set the default Django settings module for the 'celery' program.
|
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
|
|
|
app = Celery('MaxKB')
|
|
|
|
configs = {k: v for k, v in settings.__dict__.items() if k.startswith('CELERY')}
|
|
configs['worker_concurrency'] = 5
|
|
# Using a string here means the worker will not have to
|
|
# pickle the object when using Windows.
|
|
# app.config_from_object('django.conf:settings', namespace='CELERY')
|
|
|
|
configs["task_queues"] = [
|
|
Queue("celery", Exchange("celery"), routing_key="celery"),
|
|
Queue("model", Exchange("model"), routing_key="model"),
|
|
Queue("uniskb_knowledge", Exchange("uniskb_knowledge"), routing_key="uniskb_knowledge")
|
|
]
|
|
app.namespace = 'CELERY'
|
|
app.conf.update(
|
|
{key.replace('CELERY_', '') if key.replace('CELERY_', '').lower() == key.replace('CELERY_',
|
|
'') else key: configs.get(
|
|
key) for
|
|
key
|
|
in configs.keys()})
|
|
# 新知识库任务与共享 Redis 中的旧 celery 队列隔离。
|
|
knowledge_routes = {
|
|
"celery:embedding_*": {"queue": "uniskb_knowledge"},
|
|
"celery:rebuild_knowledge_index": {"queue": "uniskb_knowledge"},
|
|
"celery:finalize_rebuild_task": {"queue": "uniskb_knowledge"},
|
|
}
|
|
existing_routes = app.conf.task_routes or ()
|
|
if isinstance(existing_routes, dict):
|
|
existing_routes = (existing_routes,)
|
|
app.conf.task_routes = (knowledge_routes, *existing_routes)
|
|
app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])
|