188 lines
8.7 KiB
Python
188 lines
8.7 KiB
Python
"""Isolated task persistence and orchestration tests; no broker/model/shared DB calls."""
|
|
import unittest
|
|
import uuid
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch, Mock
|
|
from contextlib import ExitStack
|
|
from django.db import transaction, connection
|
|
from knowledge.models import KnowledgeRebuildTask, RebuildStatus, State
|
|
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
|
|
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task, MAX_POLLS
|
|
from common.exception.app_exception import AppApiException
|
|
|
|
|
|
class RebuildTests(unittest.TestCase):
|
|
def setUp(self):
|
|
with connection.cursor() as cursor:
|
|
cursor.execute('DELETE FROM knowledge_rebuild_task')
|
|
self.kid = str(uuid.uuid4())
|
|
self.docid = str(uuid.uuid4())
|
|
self.modelid = str(uuid.uuid4())
|
|
self.knowledge = SimpleNamespace(id=self.kid, embedding_model_id=self.modelid, name='test')
|
|
self.document = SimpleNamespace(id=self.docid, name='test document', status='nn2')
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
def mock(name):
|
|
return self.stack.enter_context(patch(name))
|
|
self.k = mock('knowledge.serializers.rebuild.Knowledge.objects')
|
|
self.k.select_for_update.return_value.filter.return_value.first.return_value = self.knowledge
|
|
self.k.filter.return_value.first.return_value = self.knowledge
|
|
self.docs = mock('knowledge.serializers.rebuild.Document.objects')
|
|
self.docs.filter.return_value.values_list.return_value = [self.docid]
|
|
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document])
|
|
self.paragraphs = mock('knowledge.task.rebuild.Paragraph.objects')
|
|
self.models = mock('knowledge.serializers.rebuild.Model.objects')
|
|
self.models.filter.return_value.first.return_value = SimpleNamespace(
|
|
id=self.modelid, name='embedding', model_name='test-model')
|
|
self.delay = mock('knowledge.serializers.rebuild.rebuild_knowledge_index.delay')
|
|
self.delay.return_value.id = 'queued-test'
|
|
self.poll = mock('knowledge.task.rebuild.finalize_rebuild_task.apply_async')
|
|
self.listener = mock('knowledge.task.rebuild.ListenerManagement')
|
|
self.drop = mock('knowledge.task.rebuild.drop_knowledge_index')
|
|
self.create_index = mock('knowledge.task.rebuild.create_knowledge_index')
|
|
self.index_exists = mock('knowledge.task.rebuild._index_exists')
|
|
self.index_exists.return_value = True
|
|
self.child = mock('knowledge.task.embedding.embedding_by_document.delay')
|
|
|
|
def create(self):
|
|
return KnowledgeRebuildSerializer.create('default', self.kid, None)
|
|
|
|
def task(self, status=RebuildStatus.RUNNING):
|
|
return KnowledgeRebuildTask.objects.create(
|
|
knowledge_id=self.kid, workspace_id='default', status=status, document_total=1,
|
|
config_snapshot={'embedding_model_id': self.modelid, 'document_ids': [self.docid]})
|
|
|
|
def test_dispatch_only_after_commit(self):
|
|
with transaction.atomic():
|
|
result = self.create()
|
|
self.delay.assert_not_called()
|
|
self.assertEqual(result['status'], RebuildStatus.PENDING)
|
|
self.delay.assert_called_once()
|
|
self.assertEqual(KnowledgeRebuildTask.objects.get().celery_task_id, 'queued-test')
|
|
|
|
def test_rollback_does_not_dispatch(self):
|
|
with self.assertRaises(ValueError):
|
|
with transaction.atomic():
|
|
self.create()
|
|
raise ValueError('rollback')
|
|
self.delay.assert_not_called()
|
|
self.assertFalse(KnowledgeRebuildTask.objects.exists())
|
|
|
|
def test_broker_failure_persists_failed_record(self):
|
|
self.delay.side_effect = RuntimeError('broker offline')
|
|
result = self.create()
|
|
self.assertEqual(result['status'], RebuildStatus.FAILED)
|
|
self.assertIn('broker offline', KnowledgeRebuildTask.objects.get().error_message)
|
|
|
|
def test_active_task_rejected_under_knowledge_lock(self):
|
|
self.task()
|
|
with self.assertRaises(AppApiException):
|
|
self.create()
|
|
self.k.select_for_update.assert_called_once()
|
|
self.assertEqual(KnowledgeRebuildTask.objects.count(), 1)
|
|
|
|
def test_empty_knowledge_rejected(self):
|
|
self.docs.filter.return_value.values_list.return_value = []
|
|
with self.assertRaises(AppApiException):
|
|
self.create()
|
|
self.assertFalse(KnowledgeRebuildTask.objects.exists())
|
|
|
|
def test_success_cannot_be_retried(self):
|
|
task = self.task(RebuildStatus.SUCCESS)
|
|
with self.assertRaises(AppApiException):
|
|
KnowledgeRebuildSerializer.retry('default', self.kid, str(task.id), None)
|
|
|
|
def test_failed_retry_creates_new_record(self):
|
|
old = self.task(RebuildStatus.FAILED)
|
|
result = KnowledgeRebuildSerializer.retry('default', self.kid, str(old.id), None)
|
|
self.assertNotEqual(result['id'], str(old.id))
|
|
self.assertEqual(KnowledgeRebuildTask.objects.count(), 2)
|
|
|
|
def test_duplicate_worker_delivery_does_not_delete_vectors(self):
|
|
task = self.task()
|
|
rebuild_knowledge_index.run(str(task.id))
|
|
self.listener.delete_embedding_by_knowledge.assert_not_called()
|
|
|
|
def test_worker_dispatches_independent_monitor(self):
|
|
task = self.task(RebuildStatus.PENDING)
|
|
rebuild_knowledge_index.run(str(task.id))
|
|
self.child.assert_called_once_with(self.docid, self.modelid)
|
|
self.paragraphs.filter.assert_called_once_with(knowledge_id=self.kid, document_id__in=[self.docid])
|
|
self.poll.assert_called_once()
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.RUNNING)
|
|
|
|
def test_document_dispatch_failure_is_counted(self):
|
|
task = self.task(RebuildStatus.PENDING)
|
|
self.child.side_effect = RuntimeError('cannot queue')
|
|
rebuild_knowledge_index.run(str(task.id))
|
|
self.listener.update_status.assert_any_call(
|
|
self.docs.filter.return_value, unittest.mock.ANY, State.FAILURE)
|
|
self.document.status = 'nn3'
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
|
self.assertEqual(task.document_failed, 1)
|
|
self.assertIn('cannot queue', task.error_message)
|
|
|
|
def test_success_requires_index(self):
|
|
task = self.task()
|
|
self.index_exists.return_value = False
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
|
|
|
def test_successful_finalization(self):
|
|
task = self.task()
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.SUCCESS)
|
|
self.assertEqual(task.document_success, 1)
|
|
self.assertIsNotNone(task.finished_at)
|
|
|
|
def test_pending_persists_progress_and_schedules_next_poll(self):
|
|
task = self.task()
|
|
other = SimpleNamespace(id=uuid.uuid4(), status='nn0')
|
|
task.config_snapshot['document_ids'].append(str(other.id))
|
|
task.save()
|
|
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document, other])
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.RUNNING)
|
|
self.assertEqual(task.document_success, 1)
|
|
self.poll.assert_called_once()
|
|
|
|
def test_timeout_records_failure(self):
|
|
task = self.task()
|
|
task.attempt = MAX_POLLS - 1
|
|
task.save()
|
|
self.document.status = 'nn0'
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
|
self.assertIn('超时', task.error_message)
|
|
|
|
def test_ignored_document_is_failure(self):
|
|
task = self.task()
|
|
self.document.status = 'nnn'
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.document_failed, 1)
|
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
|
|
|
def test_deleted_document_is_not_silent_success(self):
|
|
task = self.task()
|
|
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([])
|
|
finalize_rebuild_task.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.document_failed, 1)
|
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
|
|
|
def test_model_change_fails_before_destructive_work(self):
|
|
task = self.task(RebuildStatus.PENDING)
|
|
self.knowledge.embedding_model_id = str(uuid.uuid4())
|
|
rebuild_knowledge_index.run(str(task.id))
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
|
self.listener.delete_embedding_by_knowledge.assert_not_called()
|