"""Real isolated persistence and document parsing regression checks.""" import io import importlib import unittest from types import SimpleNamespace from unittest.mock import patch, Mock from django.test import TestCase from django.core.files.uploadedfile import SimpleUploadedFile from common.exception.app_exception import AppApiException from common.utils.split_model import smart_split_paragraph, SplitModel from common.utils.table_split import split_table_rows from knowledge.models import KnowledgeFolder, Knowledge from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules class MetadataTests(TestCase): def setUp(self): KnowledgeFolder.objects.create(id='default', name='root') self.k = Knowledge.objects.create(name='test', desc='', folder_id='default', meta={'internal': {'preserve': True}}) self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester') def test_metadata_persists_and_preserves_other_settings(self): Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '}) self.k.refresh_from_db() self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维') self.assertEqual(self.k.meta['internal'], {'preserve': True}) result = Metadata.one('default', str(self.k.id)) self.assertEqual(len(result['fields']), 1) self.assertEqual(result['updated_by'], '测试员') self.assertTrue(result['updated_at']) def test_partial_metadata_update_does_not_clear_other_fields(self): Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'}) Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'}) self.k.refresh_from_db() self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维') def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self): Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'}) old = Metadata.one('default', str(self.k.id)) self.assertEqual(old['fields'][0]['name'], '业务领域') Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]}) self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')]) Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]}) self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码') Metadata.update('default', str(self.k.id), self.user, {'fields': []}) self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], []) self.k.refresh_from_db() self.assertEqual(self.k.meta['internal'], {'preserve': True}) def test_custom_field_validation_does_not_write(self): cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}], [{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}], [{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}], [{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}], [{'name': str(i), 'value': ''} for i in range(51)]] for fields in cases: with self.subTest(fields=fields), self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'fields': fields}) with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'}) self.k.refresh_from_db() self.assertNotIn('business_metadata', self.k.meta) def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self): from users.models import User creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True) original_time = self.k.create_time result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]}) self.k.refresh_from_db() self.assertEqual(self.k.user_id, creator.id) self.assertEqual(self.k.create_time, original_time) self.assertEqual(result['creator_name'], '新创建者') self.assertEqual(result['user_id'], str(creator.id)) self.assertTrue(result['update_time']) self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']]) def test_rejected_pending_disabled_creators_cannot_be_selected(self): from users.models import User, UserActivationApplication creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True) application = UserActivationApplication.objects.create(user=creator, status='PENDING') for status in ('PENDING', 'REJECTED'): application.status = status; application.save() with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]}) self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']]) application.status = 'APPROVED'; application.save() creator.is_active = False; creator.save() with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)}) self.k.refresh_from_db() self.assertIsNone(self.k.user_id) self.assertNotIn('business_metadata', self.k.meta) def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self): from knowledge.serializers.knowledge import KnowledgeSerializer other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={}) operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'}) operation._validated_data = operation.initial_data operation._errors = {} params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'} with patch.object(operation, 'is_valid'): operation.edit({'embedding_params': params}, select_one=False) operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False) self.k.refresh_from_db(); other.refresh_from_db() self.assertEqual(self.k.meta['embedding_params'], params) self.assertNotIn('embedding_params', other.meta) with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException): operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False) self.k.refresh_from_db() self.assertEqual(self.k.meta['embedding_params'], params) def test_general_settings_cannot_restore_stale_metadata(self): from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]}) self.k.refresh_from_db() protected = self.k.meta['business_metadata'] operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'}) operation._validated_data = operation.initial_data operation._errors = {} with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'): operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False) self.k.refresh_from_db() self.assertEqual(self.k.meta['business_metadata'], protected) self.assertEqual(self.k.meta['selector'], 'body') self.assertNotIn('parser_id', self.k.meta) def test_invalid_types_and_unknown_fields_do_not_write(self): for data in ([1], {'business_domain': 123}, {'business_domain': False}, {'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}): with self.subTest(data=data), self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, data) self.k.refresh_from_db() self.assertNotIn('business_metadata', self.k.meta) def test_workspace_boundary(self): for fn in (Metadata.one, Rules.one): with self.assertRaises(AppApiException): fn('other', str(self.k.id)) def test_segmentation_and_metadata_updates_preserve_each_other(self): Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20}) Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'}) self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20) Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0}) self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维') def test_segmentation_rejects_invalid_and_fractional_values(self): for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10), (100,1.2), (True,0), (100,False), ('100.5',0)): with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException): Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap}) def test_upload_inherits_defaults_and_accepts_explicit_override(self): from knowledge.serializers.document import DocumentSerializers Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20}) serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'}) with patch.object(serializer,'file_to_paragraph',return_value=[]) as split: f=SimpleUploadedFile('test.txt',b'test') serializer.parse({'file':[f]}) self.assertEqual(split.call_args.args[-2:], (100,20)) serializer.parse({'file':[f],'limit':'200','overlap':'0'}) self.assertEqual(split.call_args.args[-2:], (200,0)) with self.assertRaises(AppApiException): serializer.parse({'file':[f],'limit':'100','overlap':'100'}) class ParserTests(unittest.TestCase): text=''.join(chr(0x4e00+i) for i in range(500)) def check_overlap(self, parts): self.assertGreater(len(parts),1) self.assertTrue(all(len(p)<=100 for p in parts)) self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1))) def test_plain_text_overlap(self): self.check_overlap(smart_split_paragraph(self.text,100,20)) def test_pdf_toc_and_internal_link_calls_pass_overlap(self): from common.handle.impl.text.pdf_split_handle import PdfSplitHandle doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1, load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text)) result=PdfSplitHandle.handle_toc(doc,100,20) self.check_overlap([p['content'] for p in result]) # 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。 import ast,inspect,textwrap for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links): calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method)))) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph'] self.assertTrue(calls) self.assertTrue(all(len(n.args)==3 for n in calls)) def test_table_overlap_preserves_rows_headers_and_termination(self): header='| id | value |\n| --- | --- |\n' rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)] chunks=split_table_rows(header,iter(rows),120,20) self.assertGreater(len(chunks),1) self.assertLess(len(chunks),len(rows)) self.assertTrue(all(c['content'].startswith(header) for c in chunks)) self.assertTrue(all(len(c['content'])<=120 for c in chunks)) for a,b in zip(chunks,chunks[1:]): self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:]) for row in rows:self.assertTrue(any(row in c['content'] for c in chunks)) def test_oversized_table_row_is_not_truncated(self): row='| '+'x'*500+' |\n' result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20) self.assertTrue(any(row in p['content'] for p in result)) self.assertIn('last',result[-1]['content']) def test_csv_and_xlsx_use_row_overlap(self): import openpyxl from common.handle.impl.text.csv_split_handle import CsvSplitHandle from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode() book=openpyxl.Workbook();sheet=book.active for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row) out=io.BytesIO();book.save(out) for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]: f=SimpleUploadedFile(name,data) result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20) chunks=result[0]['content'] if isinstance(result,list) else result['content'] self.assertGreater(len(chunks),1) for a,b in zip(chunks,chunks[1:]): self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:]) def test_xls_uses_row_overlap(self): from common.handle.impl.text.xls_split_handle import handle_sheet data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)] sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i]) chunks=handle_sheet('test',sheet,120,20)['content'] for a,b in zip(chunks,chunks[1:]): self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:]) def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self): migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form') model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock()) manager=Mock();manager.using.return_value.filter.return_value=[model] apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager)) editor=SimpleNamespace(connection=SimpleNamespace(alias='default')) migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor) self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p']) model.save.assert_called_once() def test_generation_parameters_reach_stream_and_nonstream_http_body(self): import json import httpx from openai import OpenAI from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel from langchain_core.messages import HumanMessage requests=[] def transport(request): body=json.loads(request.content);requests.append(body) if body.get('stream'): events=[] for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]: item={'id':'test','object':'chat.completion.chunk','model':'test','created':1, 'choices':[{'index':0,'delta':delta,'finish_reason':finish}]} events.append('data: '+json.dumps(item)+'\n\n') return httpx.Response(200,headers={'Content-Type':'text/event-stream'}, text=''.join(events)+'data: [DONE]\n\n') return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1, 'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}], 'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}}) with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http: client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http) model=BaiLianChatModel.new_instance('LLM','test',{ 'api_base':'http://example.invalid/v1','api_key':'test-only'}, temperature=0.23,top_p=0.42,max_tokens=10) model.client=client.chat.completions model.root_client=client model.streaming=False self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok') model.streaming=True self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok') self.assertEqual(len(requests),2) self.assertEqual([bool(r.get('stream')) for r in requests],[False,True]) for req in requests: self.assertEqual(req['temperature'],0.23) self.assertEqual(req['top_p'],0.42) def test_new_knowledge_tasks_use_isolated_queue(self): from ops import celery_app import knowledge.tasks for name in ('celery:embedding_by_document','celery:embedding_by_knowledge', 'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'): self.assertIn(name,celery_app.tasks) route=celery_app.amqp.router.route({},name) self.assertEqual(route['queue'].name,'uniskb_knowledge')