130 lines
5.8 KiB
Python
130 lines
5.8 KiB
Python
# coding=utf-8
|
|
"""知识库独立索引重建任务序列化器。"""
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
from rest_framework import serializers
|
|
|
|
from common.exception.app_exception import AppApiException
|
|
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document)
|
|
from knowledge.task.rebuild import rebuild_knowledge_index
|
|
from models_provider.models import Model
|
|
|
|
|
|
def _fmt(value):
|
|
if not value:
|
|
return ""
|
|
try:
|
|
value = timezone.localtime(value)
|
|
except Exception:
|
|
pass
|
|
return value.strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
def serialize_task(task: KnowledgeRebuildTask):
|
|
total = task.document_total or 0
|
|
success = task.document_success or 0
|
|
failed = task.document_failed or 0
|
|
processed = success + failed
|
|
progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0)
|
|
status_label = dict(RebuildStatus.choices).get(task.status, task.status)
|
|
return {
|
|
"id": str(task.id),
|
|
"knowledge_id": str(task.knowledge_id),
|
|
"workspace_id": task.workspace_id,
|
|
"status": task.status,
|
|
"status_label": status_label,
|
|
"config_snapshot": task.config_snapshot or {},
|
|
"document_total": total,
|
|
"document_success": success,
|
|
"document_failed": failed,
|
|
"progress": progress,
|
|
"error_message": task.error_message or "",
|
|
"created_at": _fmt(task.create_time),
|
|
"started_at": _fmt(task.started_at),
|
|
"finished_at": _fmt(task.finished_at),
|
|
}
|
|
|
|
|
|
class KnowledgeRebuildSerializer(serializers.Serializer):
|
|
@staticmethod
|
|
def create(workspace_id: str, knowledge_id: str, user_id: str):
|
|
# 锁住知识库行,串行化同一知识库的检查和任务创建。
|
|
with transaction.atomic():
|
|
knowledge = Knowledge.objects.select_for_update().filter(
|
|
id=knowledge_id, workspace_id=workspace_id
|
|
).first()
|
|
if knowledge is None:
|
|
raise AppApiException(500, "知识库不存在")
|
|
if KnowledgeRebuildTask.objects.filter(
|
|
knowledge_id=knowledge_id,
|
|
status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
|
|
).exists():
|
|
raise AppApiException(500, "该知识库已有重建任务在执行")
|
|
model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None
|
|
if model is None:
|
|
raise AppApiException(500, "知识库未配置可用的向量模型")
|
|
document_ids = [str(pk) for pk in Document.objects.filter(
|
|
knowledge_id=knowledge_id).values_list("id", flat=True)]
|
|
if not document_ids:
|
|
raise AppApiException(500, "知识库没有可重建的文档")
|
|
snapshot = {
|
|
"embedding_model_id": str(model.id),
|
|
"embedding_params": (knowledge.meta or {}).get("embedding_params", {}),
|
|
"embedding_model_name": model.name,
|
|
"embedding_model": model.model_name,
|
|
"knowledge_name": knowledge.name,
|
|
"document_ids": document_ids,
|
|
"document_total": len(document_ids),
|
|
"snapshot_at": _fmt(timezone.now()),
|
|
}
|
|
task = KnowledgeRebuildTask.objects.create(
|
|
knowledge_id=knowledge_id, workspace_id=workspace_id,
|
|
status=RebuildStatus.PENDING, config_snapshot=snapshot,
|
|
document_total=len(document_ids), operator_id=user_id,
|
|
)
|
|
|
|
def dispatch():
|
|
try:
|
|
result = rebuild_knowledge_index.delay(str(task.id))
|
|
# 不覆盖 worker 已经更新的状态。
|
|
KnowledgeRebuildTask.objects.filter(id=task.id).update(
|
|
celery_task_id=getattr(result, "id", "") or "")
|
|
except Exception as exc:
|
|
KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update(
|
|
status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}",
|
|
finished_at=timezone.now(), update_time=timezone.now())
|
|
|
|
transaction.on_commit(dispatch)
|
|
task.refresh_from_db()
|
|
return serialize_task(task)
|
|
|
|
@staticmethod
|
|
def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int):
|
|
query_set = KnowledgeRebuildTask.objects.filter(
|
|
knowledge_id=knowledge_id, workspace_id=workspace_id
|
|
).order_by("-create_time")
|
|
total = query_set.count()
|
|
start = max(0, (current_page - 1) * page_size)
|
|
records = [serialize_task(item) for item in query_set[start:start + page_size]]
|
|
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
|
|
|
|
@staticmethod
|
|
def one(workspace_id: str, knowledge_id: str, task_id: str):
|
|
task = KnowledgeRebuildTask.objects.filter(
|
|
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
|
|
).first()
|
|
if task is None:
|
|
raise AppApiException(500, "重建任务不存在")
|
|
return serialize_task(task)
|
|
|
|
@staticmethod
|
|
def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str):
|
|
old_task = KnowledgeRebuildTask.objects.filter(
|
|
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
|
|
).first()
|
|
if old_task is None:
|
|
raise AppApiException(500, "重建任务不存在")
|
|
if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL):
|
|
raise AppApiException(500, "仅失败或部分成功的任务可以重新发起")
|
|
return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id)
|