UnisKB/apps/knowledge/test_versions.py

351 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

import uuid
import json
from datetime import timedelta
from types import SimpleNamespace
from unittest import TestCase
from unittest.mock import patch
from django.db import transaction
from django.test import RequestFactory
from django.utils import timezone
from rest_framework.exceptions import ValidationError, NotFound
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
from knowledge import versions as v
from knowledge.models import (KnowledgeFolder, Knowledge, Document, Paragraph, Problem, ProblemParagraphMapping,
Tag, DocumentTag, Embedding, File, KnowledgeVersion, KnowledgeVersionLease)
from users.models import User
from models_provider.models import Model
from application.models import Application, ApplicationFolder, ApplicationVersion, ApplicationKnowledgeMapping
class VersionTests(TestCase):
def setUp(self):
self.atomic = transaction.atomic(); self.atomic.__enter__()
from django.db import connection
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
self.user = User.objects.create(username='CODEX_VERSION_TEST', nick_name='版本测试', role='ADMIN')
self.folder = KnowledgeFolder.objects.create(id='default', name='默认')
ApplicationFolder.objects.create(id='default', name='默认')
self.model = Model.objects.create(name='测试向量', model_name='test-embedding', provider='test',
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
self.root = Knowledge.objects.create(name='CODEX知识库', desc='原始', folder=self.folder, user=self.user,
embedding_model=self.model, meta={'embedding_params': {'batch_size': 8}, 'segmentation': {'chunk_size': 500}})
self.doc = Document.objects.create(knowledge=self.root, name='样例', char_length=8, status='nn2')
self.para = Paragraph.objects.create(knowledge=self.root, document=self.doc, content='旧答案:100元', position=1)
self.problem = Problem.objects.create(knowledge=self.root, content='价格多少')
ProblemParagraphMapping.objects.create(knowledge=self.root, document=self.doc, paragraph=self.para, problem=self.problem)
tag = Tag.objects.create(knowledge=self.root, key='部门', value='测试')
DocumentTag.objects.create(document=self.doc, tag=tag)
Embedding.objects.create(id=str(uuid.uuid4()), knowledge=self.root, document=self.doc, paragraph=self.para,
source_id=str(self.para.id), source_type='1', embedding='[1,0,0]', search_vector='price')
def tearDown(self):
transaction.set_rollback(True); self.atomic.__exit__(None, None, None)
def create(self, name='初版'):
return v.create_version('default', self.root.id, self.user, {'name': name})
def edit_embedding(self, values):
from knowledge.serializers.knowledge import KnowledgeSerializer
return KnowledgeSerializer.Operate(data={'user_id': str(self.user.id), 'workspace_id': 'default',
'knowledge_id': str(self.root.id)}).edit(values, select_one=False)
def test_embedding_guard_blocks_version_but_allows_unchanged_and_batch(self):
from common.exception.app_exception import AppApiException
self.create()
self.edit_embedding({'embedding_model_id': str(self.model.id), 'embedding_params': {'batch_size': 12}})
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_params': {'batch_size': 12, 'normalize': False}})
def test_embedding_guard_detects_current_and_published_assistants(self):
from common.exception.app_exception import AppApiException
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
app = Application.objects.create(name='助手', user=self.user)
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
self.edit_embedding({'embedding_model_id': str(self.model.id)})
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=other)
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
ApplicationVersion.objects.create(application=app, knowledge_setting={
'_knowledge_ids': [str(self.root.id), str(other.id)]})
ApplicationKnowledgeMapping.objects.filter(application=app, knowledge=other).delete()
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
ApplicationVersion.objects.create(application=app, knowledge_setting={'_knowledge_ids': [str(self.root.id)]})
self.edit_embedding({'embedding_model_id': str(self.model.id)})
def test_embedding_guard_detects_workflow_and_allows_single_reference(self):
from knowledge.embedding_guard import embedding_change_reason, ASSISTANT_REASON
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
node = {'type': 'search-knowledge-node', 'properties': {'node_data': {
'knowledge_id_list': [str(self.root.id)]}}}
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user,
work_flow={'nodes': [node]})
self.assertEqual(embedding_change_reason(self.root), '')
node['properties']['node_data']['knowledge_id_list'].append(str(other.id))
app.work_flow = {'nodes': [node]}; app.save()
self.assertEqual(embedding_change_reason(self.root), ASSISTANT_REASON)
def test_embedding_guard_allows_model_change_with_single_assistant(self):
other_model = Model.objects.create(name='备用向量', model_name='test-embedding-2', provider='test',
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
app = Application.objects.create(name='助手', user=self.user)
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
self.edit_embedding({'embedding_model_id': str(other_model.id)})
self.root.refresh_from_db()
self.assertEqual(self.root.embedding_model_id, other_model.id)
def test_full_snapshot_and_id_remapping(self):
version = self.create()
self.assertTrue(version.search_ready)
self.assertEqual(version.document_count, 1)
paragraph = Paragraph.objects.get(knowledge=version.snapshot)
self.assertNotEqual(paragraph.id, self.para.id)
self.assertEqual(paragraph.content, self.para.content)
mapping = ProblemParagraphMapping.objects.get(knowledge=version.snapshot)
self.assertEqual(mapping.paragraph_id, paragraph.id)
self.assertEqual(mapping.document_id, paragraph.document_id)
self.assertEqual(DocumentTag.objects.filter(document__knowledge=version.snapshot).count(), 1)
vector = Embedding.objects.get(knowledge=version.snapshot)
self.assertEqual(vector.paragraph_id, paragraph.id)
self.assertEqual(vector.source_id, str(paragraph.id))
def test_live_edits_do_not_change_snapshot(self):
version = self.create()
self.para.content = '新答案:200元'; self.para.save()
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, '旧答案:100元')
second = self.create('第二版')
self.assertEqual(Paragraph.objects.get(knowledge=second.snapshot).content, '新答案:200元')
self.assertNotEqual(version.content_hash, second.content_hash)
def test_text_uuid_is_preserved(self):
self.para.content = '业务标识:' + str(self.doc.id); self.para.save()
version = self.create()
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, self.para.content)
def test_binary_files_shared_and_markdown_rewritten(self):
file = File(id=uuid.uuid4(), file_name='测试.png', loid=12345, source_id=str(self.root.id),
source_type='KNOWLEDGE', sha256_hash='test', file_size=10, meta={'knowledge_id': str(self.root.id)})
File.objects.bulk_create([file])
self.para.content = '![](./oss/file/' + str(file.id) + ')'; self.para.save()
version = self.create()
cloned = File.objects.get(source_id=str(version.snapshot_id))
self.assertEqual(cloned.loid, file.loid)
self.assertNotEqual(cloned.id, file.id)
self.assertIn(str(cloned.id), Paragraph.objects.get(knowledge=version.snapshot).content)
with patch('knowledge.models.knowledge.select_one') as unlink:
v.delete_version('default', self.root.id, version.id)
unlink.assert_not_called()
self.assertTrue(File.objects.filter(pk=file.id).exists())
def test_version_delete_cleans_only_snapshot(self):
version = self.create(); snapshot_id = version.snapshot_id
v.delete_version('default', self.root.id, version.id)
for model in (Document, Paragraph, Problem, Embedding, Tag, ProblemParagraphMapping):
self.assertFalse(model.objects.filter(knowledge_id=snapshot_id).exists())
self.assertFalse(Knowledge.objects.filter(pk=snapshot_id).exists())
self.assertTrue(Paragraph.objects.filter(pk=self.para.id).exists())
def test_version_number_not_reused(self):
first = self.create(); v.delete_version('default', self.root.id, first.id)
self.assertEqual(self.create().number, 2)
def test_create_transaction_rolls_back_on_index_failure(self):
with patch.object(v, '_build_index', side_effect=RuntimeError('index failed')), self.assertRaises(RuntimeError): self.create()
self.assertEqual(Knowledge.objects.count(), 1)
self.assertEqual(KnowledgeVersion.objects.count(), 0)
def test_pending_processing_rejected(self):
self.doc.status = 'nn0'; self.doc.save()
with self.assertRaises(ValidationError): self.create()
def test_missing_vectors_are_browsable_but_not_selectable(self):
Embedding.objects.all().delete(); version = self.create()
self.assertFalse(version.search_ready)
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [self.root.id])
def test_cross_knowledge_selection_rejected(self):
version = self.create()
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [])
def test_invalid_version_rejected(self):
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): 'invalid'}, 'default', [self.root.id])
def test_cross_workspace_rejected(self):
version = self.create()
with self.assertRaises(NotFound404): v.get_version('other', self.root.id, version.id)
def test_fixed_selection_and_live_lookup(self):
version = self.create()
self.assertEqual(v.resolve_search([self.root.id], {}, 'default'), [str(self.root.id)])
self.assertEqual(v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default'), [str(version.snapshot_id)])
def test_document_scope_maps_to_frozen_documents(self):
version = self.create(); document = Document.objects.get(knowledge=version.snapshot)
self.assertEqual(v.resolve_document_scope([self.doc.id], {str(self.root.id): str(version.id)}, 'default'), [str(document.id)])
self.assertEqual(v.resolve_document_scope([self.doc.id], {}, 'default'), [str(self.doc.id)])
data = v.version_data(version)
self.assertNotIn('_document_ids', data['config'])
self.assertNotIn('model_identity', data['config'])
def test_same_chat_keeps_latest_choice(self):
first = self.create()
selection = {str(self.root.id): 'latest'}
pinned = v.pin_selection(selection, 'default', 'chat-test')
self.create('新版')
self.assertEqual(v.pin_selection(selection, 'default', 'chat-test'), pinned)
self.assertEqual(pinned[str(self.root.id)], str(first.id))
self.assertNotEqual(v.pin_selection(selection, 'default', 'new-chat'), pinned)
def test_active_conversation_protects_delete(self):
version = self.create(); v.pin_selection({str(self.root.id): str(version.id)}, 'default', 'chat-test')
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
KnowledgeVersionLease.objects.update(expires_at=timezone.now()-timedelta(seconds=1))
v.delete_version('default', self.root.id, version.id)
def test_assistant_pin_protects_delete(self):
version = self.create()
app = Application.objects.create(name='测试助手', user=self.user,
knowledge_setting={'version_selection': {str(self.root.id): str(version.id)}})
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
app.knowledge_setting = {}; app.save(); v.delete_version('default', self.root.id, version.id)
def test_latest_assistant_protects_only_latest_ready_version(self):
first = self.create(); second = self.create('新版')
Application.objects.create(name='测试助手', user=self.user,
knowledge_setting={'version_selection': {str(self.root.id): 'latest'}})
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, second.id)
v.delete_version('default', self.root.id, first.id)
def test_workflow_draft_and_published_version_protect_delete(self):
version = self.create()
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
'knowledge_id_list': [str(self.root.id)],
'knowledge_setting': {'version_selection': {str(self.root.id): str(version.id)}}}}}]}
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user, work_flow=flow)
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
app.work_flow = {}; app.save()
published = ApplicationVersion.objects.create(application=app, type='WORK_FLOW', work_flow=flow)
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
published.delete(); v.delete_version('default', self.root.id, version.id)
def test_workflow_conflicting_versions_rejected(self):
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
'knowledge_setting': {'version_selection': {str(self.root.id): 'latest'}}}}}]}
with self.assertRaises(ValidationError):
v.application_selection({'version_selection': {str(self.root.id): 'different'}}, flow)
def test_republication_releases_superseded_version(self):
version = self.create(); app = Application.objects.create(name='重新发布', user=self.user)
ApplicationVersion.objects.create(application=app, knowledge_setting={
'version_selection': {str(self.root.id): str(version.id)}})
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
ApplicationVersion.objects.create(application=app, knowledge_setting={})
v.delete_version('default', self.root.id, version.id)
def test_migration_target_cannot_be_snapshot(self):
version = self.create(); request = RequestFactory().post('/document/migrate')
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(self.root.id),
'target_knowledge_id': str(version.snapshot_id)})
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
def test_published_chat_keeps_published_association(self):
from application.serializers.common import ChatInfo
version = self.create()
app = Application.objects.create(name='发布测试', user=self.user)
ApplicationVersion.objects.create(application=app, knowledge_setting={
'_knowledge_ids': [str(self.root.id)], 'version_selection': {str(self.root.id): str(version.id)}})
self.assertFalse(ApplicationKnowledgeMapping.objects.filter(application=app).exists())
chat = ChatInfo(str(uuid.uuid4()), str(self.user.id), 'PLATFORM_USER', [], [], str(app.id))
chat.get_application()
self.assertEqual(chat.knowledge_id_list, [str(self.root.id)])
self.assertEqual(chat.knowledge_version_map, {str(self.root.id): str(version.id)})
def test_feedback_body_cannot_write_snapshot(self):
from application.serializers.application_chat_record import ApplicationChatRecordAddKnowledgeSerializer
version = self.create(); doc = Document.objects.get(knowledge=version.snapshot)
app = Application.objects.create(name='改进测试', user=self.user)
serializer = ApplicationChatRecordAddKnowledgeSerializer(data={'workspace_id': 'default',
'application_id': str(app.id), 'knowledge_id': str(version.snapshot_id),
'document_id': str(doc.id), 'chat_ids': []})
with self.assertRaises(NotFound404): serializer.post_improve({}, with_valid=False)
def test_model_changes_fail_explicitly(self):
version = self.create(); self.model.model_name = 'different'; self.model.save()
with self.assertRaises(ValidationError): v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default')
def test_internal_snapshot_api_guard(self):
version = self.create(); request = RequestFactory().get('/admin/api/knowledge')
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(version.snapshot_id)})
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
with self.assertRaises(NotFound404): v.root_knowledge('default', version.snapshot_id)
def test_api_read_permission_and_workspace(self):
from rest_framework.test import APIRequestFactory, force_authenticate
from knowledge.views.version import KnowledgeVersionView
self.create(); factory = APIRequestFactory()
for roles, workspace, expected in [([], 'default', 403), (['ADMIN'], 'default', 200), (['ADMIN'], 'other', 404)]:
req = factory.get('/versions')
force_authenticate(req, user=self.user, token=SimpleNamespace(role_list=roles, permission_list=[]))
response = KnowledgeVersionView.as_view()(req, workspace_id=workspace, knowledge_id=self.root.id)
self.assertEqual(response.status_code, expected)
def test_empty_name_rejected(self):
with self.assertRaises(ValidationError): self.create(' ')
def test_real_postgres_retrieval_reads_frozen_content(self):
from django.db import connection
if connection.vendor != 'postgresql':
self.skipTest('Real pgvector retrieval requires isolated PostgreSQL schema')
from application.chat_pipeline.step.search_dataset_step.impl import base_search_dataset_step as search
version = self.create()
self.para.content = '新答案:200元'; self.para.save()
manage = SimpleNamespace(context={'knowledge_version_map': {str(self.root.id): str(version.id)}, 'chat_user_type': 'PLATFORM_USER'})
model = SimpleNamespace(model_type='EMBEDDING', name='测试向量')
embedding = SimpleNamespace(embed_query=lambda text: [1.0, 0.0, 0.0])
with patch.object(search, 'get_model_by_id', return_value=model), patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding):
result = search.BaseSearchDatasetStep().execute('价格', [str(self.root.id)], [], [], 3, 0.1,
search_mode='embedding', workspace_id='default', manage=manage)
self.assertEqual(len(result), 1)
self.assertEqual(result[0].content, '旧答案:100元')
self.assertIn('v1', result[0].knowledge_name)
def test_real_postgres_workflow_retrieves_frozen_document(self):
from django.db import connection
if connection.vendor != 'postgresql': self.skipTest('Real workflow pgvector retrieval requires PostgreSQL')
from application.flow.step_node.search_knowledge_node.impl import base_search_knowledge_node as search
version = self.create(); self.para.content = '新答案:200元'; self.para.save()
body = {'knowledge_version_map': {str(self.root.id): str(version.id)},
'chat_user_type': 'PLATFORM_USER', 'workspace_id': 'default'}
node = search.BaseSearchKnowledgeNode.__new__(search.BaseSearchKnowledgeNode)
node.context = {}; node.workflow_manage = SimpleNamespace(get_body=lambda: body,
get_reference_field=lambda *_: [str(self.doc.id)])
embedding = SimpleNamespace(embed_query=lambda _: [1.0, 0.0, 0.0])
with patch.object(search, 'get_model_instance_by_model_workspace_id'), \
patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding), \
patch.object(search.connection, 'close'):
result = node.execute([str(self.root.id)], {'top_n':3, 'similarity':0.1, 'search_mode':'embedding'},
'价格', True, 'referencing', 'document', ['previous', 'documents'])
self.assertEqual(result.node_variable['paragraph_list'][0]['content'], '旧答案:100元')
def test_api_create_browse_and_delete(self):
from rest_framework.test import APIRequestFactory, force_authenticate
from knowledge.views.version import KnowledgeVersionView
factory = APIRequestFactory()
def call(method, data=None, version_id=None):
request = getattr(factory, method)('/versions', data=data or {}, format='json')
force_authenticate(request, user=self.user, token=SimpleNamespace(role_list=['ADMIN'], permission_list=[]))
kw = {'workspace_id': 'default', 'knowledge_id': self.root.id}
if version_id: kw['version_id'] = version_id
return KnowledgeVersionView.as_view()(request, **kw)
created = call('post', {'name': '接口创建'})
self.assertEqual(json.loads(created.content)['code'], 200)
version_id = json.loads(created.content)['data']['id']
detail = call('get', version_id=version_id)
self.assertEqual(json.loads(detail.content)['data']['documents'][0]['name'], self.doc.name)
removed = call('delete', version_id=version_id)
self.assertEqual(json.loads(removed.content)['code'], 200)