UnisKB/apps/knowledge/task/rebuild.py

147 lines
8.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

# coding=utf-8
"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。"""
from django.db import transaction
from django.utils import timezone
from common.db.sql_execute import sql_execute
from common.event.listener_manage import ListenerManagement
from common.utils.logger import maxkb_logger
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph,
TaskType, State, Status)
from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index
from ops import celery_app
POLL_SECONDS = 5
MAX_POLLS = 120
def _index_exists(knowledge_id: str) -> bool:
rows = sql_execute(
"SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s",
[f"embedding_hnsw_idx_{knowledge_id}"]
)
return bool(rows)
def _fail(task_id, message):
KnowledgeRebuildTask.objects.filter(
id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
).update(status=RebuildStatus.FAILED, error_message=message,
finished_at=timezone.now(), update_time=timezone.now())
@celery_app.task(name="celery:rebuild_knowledge_index")
def rebuild_knowledge_index(task_id: str):
# 重复投递不能重复清空向量或重复派发。
claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update(
status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="",
attempt=0, update_time=timezone.now())
if not claimed:
return
try:
task = KnowledgeRebuildTask.objects.get(id=task_id)
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
if knowledge is None:
raise ValueError("知识库不存在")
snapshot = task.config_snapshot or {}
model_id = snapshot.get("embedding_model_id")
if not model_id or str(knowledge.embedding_model_id) != model_id:
raise ValueError("向量模型已变更,请重新发起任务")
if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}):
raise ValueError('嵌入参数已变更,请重新发起任务')
# 兼容历史任务;新任务在受理时即固定文档范围。
document_ids = snapshot.get("document_ids")
if document_ids is None:
document_ids = [str(pk) for pk in Document.objects.filter(
knowledge_id=knowledge.id).values_list("id", flat=True)]
snapshot["document_ids"] = document_ids
task.config_snapshot = snapshot
task.save(update_fields=["config_snapshot", "update_time"])
documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids))
if not documents or len(documents) != len(document_ids):
raise ValueError("重建范围为空或文档已删除,请重新发起任务")
if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents):
raise ValueError("文档正在向量化,请完成后重新发起")
ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id))
drop_knowledge_index(knowledge_id=str(knowledge.id))
task.document_total = len(document_ids)
task.save(update_fields=["document_total", "update_time"])
# 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。
target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)
ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING)
ListenerManagement.update_status(
Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids),
TaskType.EMBEDDING, State.PENDING)
from knowledge.task.embedding import embedding_by_document
dispatch_errors = []
for document in documents:
try:
embedding_by_document.delay(str(document.id), model_id)
except Exception as exc:
ListenerManagement.update_status(
Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE)
dispatch_errors.append(f"{document.name}:{exc}")
if dispatch_errors:
task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000]
task.save(update_fields=["error_message", "update_time"])
# 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。
finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS)
except Exception as exc:
_fail(task_id, f"重建任务启动失败:{exc}")
maxkb_logger.exception("rebuild knowledge index failed: %s", task_id)
@celery_app.task(name="celery:finalize_rebuild_task")
def finalize_rebuild_task(task_id: str):
try:
with transaction.atomic():
task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first()
if task is None or task.status != RebuildStatus.RUNNING:
return
snapshot = task.config_snapshot or {}
document_ids = snapshot.get("document_ids", [])
documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids))
states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents]
success = states.count(State.SUCCESS)
failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED)
for state in states) + len(document_ids) - len(documents)
pending = len(document_ids) - success - failed
task.document_total = len(document_ids)
task.document_success = success
task.document_failed = failed
if pending:
task.attempt += 1
if task.attempt < MAX_POLLS:
task.save(update_fields=["document_total", "document_success", "document_failed",
"attempt", "update_time"])
transaction.on_commit(lambda: finalize_rebuild_task.apply_async(
(task_id,), countdown=POLL_SECONDS))
return
task.status = RebuildStatus.FAILED
task.error_message = f"重建超时:仍有 {pending} 个文档未完成"
else:
knowledge_id = str(task.knowledge_id)
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or
snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})):
raise ValueError("向量模型已变更,请重新发起任务")
ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)()
create_knowledge_index(knowledge_id=knowledge_id)
index_ready = _index_exists(knowledge_id)
if success and failed == 0 and index_ready:
task.status = RebuildStatus.SUCCESS
task.error_message = ""
elif success and failed and index_ready:
task.status = RebuildStatus.PARTIAL
task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message
else:
task.status = RebuildStatus.FAILED
task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message
task.finished_at = timezone.now()
task.save(update_fields=["document_total", "document_success", "document_failed", "status",
"error_message", "finished_at", "attempt", "update_time"])
except Exception as exc:
_fail(task_id, f"重建结果汇总失败:{exc}")
maxkb_logger.exception("finalize rebuild task failed: %s", task_id)