UnisKB/apps/knowledge/serializers/rebuild.py

130 lines
5.8 KiB
Python

# coding=utf-8
"""知识库独立索引重建任务序列化器。"""
from django.db import transaction
from django.utils import timezone
from rest_framework import serializers
from common.exception.app_exception import AppApiException
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document)
from knowledge.task.rebuild import rebuild_knowledge_index
from models_provider.models import Model
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def serialize_task(task: KnowledgeRebuildTask):
total = task.document_total or 0
success = task.document_success or 0
failed = task.document_failed or 0
processed = success + failed
progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0)
status_label = dict(RebuildStatus.choices).get(task.status, task.status)
return {
"id": str(task.id),
"knowledge_id": str(task.knowledge_id),
"workspace_id": task.workspace_id,
"status": task.status,
"status_label": status_label,
"config_snapshot": task.config_snapshot or {},
"document_total": total,
"document_success": success,
"document_failed": failed,
"progress": progress,
"error_message": task.error_message or "",
"created_at": _fmt(task.create_time),
"started_at": _fmt(task.started_at),
"finished_at": _fmt(task.finished_at),
}
class KnowledgeRebuildSerializer(serializers.Serializer):
@staticmethod
def create(workspace_id: str, knowledge_id: str, user_id: str):
# 锁住知识库行,串行化同一知识库的检查和任务创建。
with transaction.atomic():
knowledge = Knowledge.objects.select_for_update().filter(
id=knowledge_id, workspace_id=workspace_id
).first()
if knowledge is None:
raise AppApiException(500, "知识库不存在")
if KnowledgeRebuildTask.objects.filter(
knowledge_id=knowledge_id,
status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
).exists():
raise AppApiException(500, "该知识库已有重建任务在执行")
model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None
if model is None:
raise AppApiException(500, "知识库未配置可用的向量模型")
document_ids = [str(pk) for pk in Document.objects.filter(
knowledge_id=knowledge_id).values_list("id", flat=True)]
if not document_ids:
raise AppApiException(500, "知识库没有可重建的文档")
snapshot = {
"embedding_model_id": str(model.id),
"embedding_params": (knowledge.meta or {}).get("embedding_params", {}),
"embedding_model_name": model.name,
"embedding_model": model.model_name,
"knowledge_name": knowledge.name,
"document_ids": document_ids,
"document_total": len(document_ids),
"snapshot_at": _fmt(timezone.now()),
}
task = KnowledgeRebuildTask.objects.create(
knowledge_id=knowledge_id, workspace_id=workspace_id,
status=RebuildStatus.PENDING, config_snapshot=snapshot,
document_total=len(document_ids), operator_id=user_id,
)
def dispatch():
try:
result = rebuild_knowledge_index.delay(str(task.id))
# 不覆盖 worker 已经更新的状态。
KnowledgeRebuildTask.objects.filter(id=task.id).update(
celery_task_id=getattr(result, "id", "") or "")
except Exception as exc:
KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update(
status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}",
finished_at=timezone.now(), update_time=timezone.now())
transaction.on_commit(dispatch)
task.refresh_from_db()
return serialize_task(task)
@staticmethod
def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int):
query_set = KnowledgeRebuildTask.objects.filter(
knowledge_id=knowledge_id, workspace_id=workspace_id
).order_by("-create_time")
total = query_set.count()
start = max(0, (current_page - 1) * page_size)
records = [serialize_task(item) for item in query_set[start:start + page_size]]
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
@staticmethod
def one(workspace_id: str, knowledge_id: str, task_id: str):
task = KnowledgeRebuildTask.objects.filter(
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
).first()
if task is None:
raise AppApiException(500, "重建任务不存在")
return serialize_task(task)
@staticmethod
def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str):
old_task = KnowledgeRebuildTask.objects.filter(
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
).first()
if old_task is None:
raise AppApiException(500, "重建任务不存在")
if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL):
raise AppApiException(500, "仅失败或部分成功的任务可以重新发起")
return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id)