147 lines
8.0 KiB
Python
147 lines
8.0 KiB
Python
# coding=utf-8
|
||
"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。"""
|
||
from django.db import transaction
|
||
from django.utils import timezone
|
||
|
||
from common.db.sql_execute import sql_execute
|
||
from common.event.listener_manage import ListenerManagement
|
||
from common.utils.logger import maxkb_logger
|
||
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph,
|
||
TaskType, State, Status)
|
||
from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index
|
||
from ops import celery_app
|
||
|
||
POLL_SECONDS = 5
|
||
MAX_POLLS = 120
|
||
|
||
|
||
def _index_exists(knowledge_id: str) -> bool:
|
||
rows = sql_execute(
|
||
"SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s",
|
||
[f"embedding_hnsw_idx_{knowledge_id}"]
|
||
)
|
||
return bool(rows)
|
||
|
||
|
||
def _fail(task_id, message):
|
||
KnowledgeRebuildTask.objects.filter(
|
||
id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
|
||
).update(status=RebuildStatus.FAILED, error_message=message,
|
||
finished_at=timezone.now(), update_time=timezone.now())
|
||
|
||
|
||
@celery_app.task(name="celery:rebuild_knowledge_index")
|
||
def rebuild_knowledge_index(task_id: str):
|
||
# 重复投递不能重复清空向量或重复派发。
|
||
claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update(
|
||
status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="",
|
||
attempt=0, update_time=timezone.now())
|
||
if not claimed:
|
||
return
|
||
try:
|
||
task = KnowledgeRebuildTask.objects.get(id=task_id)
|
||
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
|
||
if knowledge is None:
|
||
raise ValueError("知识库不存在")
|
||
snapshot = task.config_snapshot or {}
|
||
model_id = snapshot.get("embedding_model_id")
|
||
if not model_id or str(knowledge.embedding_model_id) != model_id:
|
||
raise ValueError("向量模型已变更,请重新发起任务")
|
||
if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}):
|
||
raise ValueError('嵌入参数已变更,请重新发起任务')
|
||
# 兼容历史任务;新任务在受理时即固定文档范围。
|
||
document_ids = snapshot.get("document_ids")
|
||
if document_ids is None:
|
||
document_ids = [str(pk) for pk in Document.objects.filter(
|
||
knowledge_id=knowledge.id).values_list("id", flat=True)]
|
||
snapshot["document_ids"] = document_ids
|
||
task.config_snapshot = snapshot
|
||
task.save(update_fields=["config_snapshot", "update_time"])
|
||
documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids))
|
||
if not documents or len(documents) != len(document_ids):
|
||
raise ValueError("重建范围为空或文档已删除,请重新发起任务")
|
||
if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents):
|
||
raise ValueError("文档正在向量化,请完成后重新发起")
|
||
|
||
ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id))
|
||
drop_knowledge_index(knowledge_id=str(knowledge.id))
|
||
task.document_total = len(document_ids)
|
||
task.save(update_fields=["document_total", "update_time"])
|
||
# 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。
|
||
target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)
|
||
ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING)
|
||
ListenerManagement.update_status(
|
||
Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids),
|
||
TaskType.EMBEDDING, State.PENDING)
|
||
from knowledge.task.embedding import embedding_by_document
|
||
dispatch_errors = []
|
||
for document in documents:
|
||
try:
|
||
embedding_by_document.delay(str(document.id), model_id)
|
||
except Exception as exc:
|
||
ListenerManagement.update_status(
|
||
Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE)
|
||
dispatch_errors.append(f"{document.name}:{exc}")
|
||
if dispatch_errors:
|
||
task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000]
|
||
task.save(update_fields=["error_message", "update_time"])
|
||
# 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。
|
||
finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS)
|
||
except Exception as exc:
|
||
_fail(task_id, f"重建任务启动失败:{exc}")
|
||
maxkb_logger.exception("rebuild knowledge index failed: %s", task_id)
|
||
|
||
|
||
@celery_app.task(name="celery:finalize_rebuild_task")
|
||
def finalize_rebuild_task(task_id: str):
|
||
try:
|
||
with transaction.atomic():
|
||
task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first()
|
||
if task is None or task.status != RebuildStatus.RUNNING:
|
||
return
|
||
snapshot = task.config_snapshot or {}
|
||
document_ids = snapshot.get("document_ids", [])
|
||
documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids))
|
||
states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents]
|
||
success = states.count(State.SUCCESS)
|
||
failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED)
|
||
for state in states) + len(document_ids) - len(documents)
|
||
pending = len(document_ids) - success - failed
|
||
task.document_total = len(document_ids)
|
||
task.document_success = success
|
||
task.document_failed = failed
|
||
if pending:
|
||
task.attempt += 1
|
||
if task.attempt < MAX_POLLS:
|
||
task.save(update_fields=["document_total", "document_success", "document_failed",
|
||
"attempt", "update_time"])
|
||
transaction.on_commit(lambda: finalize_rebuild_task.apply_async(
|
||
(task_id,), countdown=POLL_SECONDS))
|
||
return
|
||
task.status = RebuildStatus.FAILED
|
||
task.error_message = f"重建超时:仍有 {pending} 个文档未完成"
|
||
else:
|
||
knowledge_id = str(task.knowledge_id)
|
||
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
|
||
if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or
|
||
snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})):
|
||
raise ValueError("向量模型已变更,请重新发起任务")
|
||
ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)()
|
||
create_knowledge_index(knowledge_id=knowledge_id)
|
||
index_ready = _index_exists(knowledge_id)
|
||
if success and failed == 0 and index_ready:
|
||
task.status = RebuildStatus.SUCCESS
|
||
task.error_message = ""
|
||
elif success and failed and index_ready:
|
||
task.status = RebuildStatus.PARTIAL
|
||
task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message
|
||
else:
|
||
task.status = RebuildStatus.FAILED
|
||
task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message
|
||
task.finished_at = timezone.now()
|
||
task.save(update_fields=["document_total", "document_success", "document_failed", "status",
|
||
"error_message", "finished_at", "attempt", "update_time"])
|
||
except Exception as exc:
|
||
_fail(task_id, f"重建结果汇总失败:{exc}")
|
||
maxkb_logger.exception("finalize rebuild task failed: %s", task_id)
|