289 lines
18 KiB
Python
289 lines
18 KiB
Python
"""Real isolated persistence and document parsing regression checks."""
|
||
import io
|
||
import importlib
|
||
import unittest
|
||
from types import SimpleNamespace
|
||
from unittest.mock import patch, Mock
|
||
from django.test import TestCase
|
||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||
from common.exception.app_exception import AppApiException
|
||
from common.utils.split_model import smart_split_paragraph, SplitModel
|
||
from common.utils.table_split import split_table_rows
|
||
from knowledge.models import KnowledgeFolder, Knowledge
|
||
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules
|
||
|
||
|
||
class MetadataTests(TestCase):
|
||
def setUp(self):
|
||
KnowledgeFolder.objects.create(id='default', name='root')
|
||
self.k = Knowledge.objects.create(name='test', desc='', folder_id='default',
|
||
meta={'internal': {'preserve': True}})
|
||
self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester')
|
||
|
||
def test_metadata_persists_and_preserves_other_settings(self):
|
||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '})
|
||
self.k.refresh_from_db()
|
||
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
|
||
self.assertEqual(self.k.meta['internal'], {'preserve': True})
|
||
result = Metadata.one('default', str(self.k.id))
|
||
self.assertEqual(len(result['fields']), 1)
|
||
self.assertEqual(result['updated_by'], '测试员')
|
||
self.assertTrue(result['updated_at'])
|
||
|
||
def test_partial_metadata_update_does_not_clear_other_fields(self):
|
||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
|
||
Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'})
|
||
self.k.refresh_from_db()
|
||
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
|
||
|
||
def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self):
|
||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'})
|
||
old = Metadata.one('default', str(self.k.id))
|
||
self.assertEqual(old['fields'][0]['name'], '业务领域')
|
||
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]})
|
||
self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')])
|
||
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]})
|
||
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码')
|
||
Metadata.update('default', str(self.k.id), self.user, {'fields': []})
|
||
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], [])
|
||
self.k.refresh_from_db()
|
||
self.assertEqual(self.k.meta['internal'], {'preserve': True})
|
||
|
||
def test_custom_field_validation_does_not_write(self):
|
||
cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}],
|
||
[{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}],
|
||
[{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}],
|
||
[{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}],
|
||
[{'name': str(i), 'value': ''} for i in range(51)]]
|
||
for fields in cases:
|
||
with self.subTest(fields=fields), self.assertRaises(AppApiException):
|
||
Metadata.update('default', str(self.k.id), self.user, {'fields': fields})
|
||
with self.assertRaises(AppApiException):
|
||
Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'})
|
||
self.k.refresh_from_db()
|
||
self.assertNotIn('business_metadata', self.k.meta)
|
||
|
||
def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self):
|
||
from users.models import User
|
||
creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True)
|
||
original_time = self.k.create_time
|
||
result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
|
||
self.k.refresh_from_db()
|
||
self.assertEqual(self.k.user_id, creator.id)
|
||
self.assertEqual(self.k.create_time, original_time)
|
||
self.assertEqual(result['creator_name'], '新创建者')
|
||
self.assertEqual(result['user_id'], str(creator.id))
|
||
self.assertTrue(result['update_time'])
|
||
self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']])
|
||
|
||
def test_rejected_pending_disabled_creators_cannot_be_selected(self):
|
||
from users.models import User, UserActivationApplication
|
||
creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True)
|
||
application = UserActivationApplication.objects.create(user=creator, status='PENDING')
|
||
for status in ('PENDING', 'REJECTED'):
|
||
application.status = status; application.save()
|
||
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
|
||
self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']])
|
||
application.status = 'APPROVED'; application.save()
|
||
creator.is_active = False; creator.save()
|
||
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)})
|
||
self.k.refresh_from_db()
|
||
self.assertIsNone(self.k.user_id)
|
||
self.assertNotIn('business_metadata', self.k.meta)
|
||
|
||
def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self):
|
||
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||
other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={})
|
||
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
|
||
operation._validated_data = operation.initial_data
|
||
operation._errors = {}
|
||
params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'}
|
||
with patch.object(operation, 'is_valid'):
|
||
operation.edit({'embedding_params': params}, select_one=False)
|
||
operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False)
|
||
self.k.refresh_from_db(); other.refresh_from_db()
|
||
self.assertEqual(self.k.meta['embedding_params'], params)
|
||
self.assertNotIn('embedding_params', other.meta)
|
||
with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException):
|
||
operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False)
|
||
self.k.refresh_from_db()
|
||
self.assertEqual(self.k.meta['embedding_params'], params)
|
||
|
||
def test_general_settings_cannot_restore_stale_metadata(self):
|
||
from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest
|
||
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]})
|
||
self.k.refresh_from_db()
|
||
protected = self.k.meta['business_metadata']
|
||
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
|
||
operation._validated_data = operation.initial_data
|
||
operation._errors = {}
|
||
with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'):
|
||
operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False)
|
||
self.k.refresh_from_db()
|
||
self.assertEqual(self.k.meta['business_metadata'], protected)
|
||
self.assertEqual(self.k.meta['selector'], 'body')
|
||
self.assertNotIn('parser_id', self.k.meta)
|
||
|
||
def test_invalid_types_and_unknown_fields_do_not_write(self):
|
||
for data in ([1], {'business_domain': 123}, {'business_domain': False},
|
||
{'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}):
|
||
with self.subTest(data=data), self.assertRaises(AppApiException):
|
||
Metadata.update('default', str(self.k.id), self.user, data)
|
||
self.k.refresh_from_db()
|
||
self.assertNotIn('business_metadata', self.k.meta)
|
||
|
||
def test_workspace_boundary(self):
|
||
for fn in (Metadata.one, Rules.one):
|
||
with self.assertRaises(AppApiException):
|
||
fn('other', str(self.k.id))
|
||
|
||
def test_segmentation_and_metadata_updates_preserve_each_other(self):
|
||
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20})
|
||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
|
||
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20)
|
||
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0})
|
||
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维')
|
||
|
||
def test_segmentation_rejects_invalid_and_fractional_values(self):
|
||
for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10),
|
||
(100,1.2), (True,0), (100,False), ('100.5',0)):
|
||
with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException):
|
||
Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap})
|
||
|
||
def test_upload_inherits_defaults_and_accepts_explicit_override(self):
|
||
from knowledge.serializers.document import DocumentSerializers
|
||
Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20})
|
||
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
|
||
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
|
||
f=SimpleUploadedFile('test.txt',b'test')
|
||
serializer.parse({'file':[f]})
|
||
self.assertEqual(split.call_args.args[-2:], (100,20))
|
||
serializer.parse({'file':[f],'limit':'200','overlap':'0'})
|
||
self.assertEqual(split.call_args.args[-2:], (200,0))
|
||
with self.assertRaises(AppApiException):
|
||
serializer.parse({'file':[f],'limit':'100','overlap':'100'})
|
||
|
||
|
||
class ParserTests(unittest.TestCase):
|
||
text=''.join(chr(0x4e00+i) for i in range(500))
|
||
|
||
def check_overlap(self, parts):
|
||
self.assertGreater(len(parts),1)
|
||
self.assertTrue(all(len(p)<=100 for p in parts))
|
||
self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1)))
|
||
|
||
def test_plain_text_overlap(self):
|
||
self.check_overlap(smart_split_paragraph(self.text,100,20))
|
||
|
||
def test_pdf_toc_and_internal_link_calls_pass_overlap(self):
|
||
from common.handle.impl.text.pdf_split_handle import PdfSplitHandle
|
||
doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1,
|
||
load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text))
|
||
result=PdfSplitHandle.handle_toc(doc,100,20)
|
||
self.check_overlap([p['content'] for p in result])
|
||
# 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。
|
||
import ast,inspect,textwrap
|
||
for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links):
|
||
calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method))))
|
||
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph']
|
||
self.assertTrue(calls)
|
||
self.assertTrue(all(len(n.args)==3 for n in calls))
|
||
|
||
def test_table_overlap_preserves_rows_headers_and_termination(self):
|
||
header='| id | value |\n| --- | --- |\n'
|
||
rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)]
|
||
chunks=split_table_rows(header,iter(rows),120,20)
|
||
self.assertGreater(len(chunks),1)
|
||
self.assertLess(len(chunks),len(rows))
|
||
self.assertTrue(all(c['content'].startswith(header) for c in chunks))
|
||
self.assertTrue(all(len(c['content'])<=120 for c in chunks))
|
||
for a,b in zip(chunks,chunks[1:]):
|
||
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||
for row in rows:self.assertTrue(any(row in c['content'] for c in chunks))
|
||
|
||
def test_oversized_table_row_is_not_truncated(self):
|
||
row='| '+'x'*500+' |\n'
|
||
result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20)
|
||
self.assertTrue(any(row in p['content'] for p in result))
|
||
self.assertIn('last',result[-1]['content'])
|
||
|
||
def test_csv_and_xlsx_use_row_overlap(self):
|
||
import openpyxl
|
||
from common.handle.impl.text.csv_split_handle import CsvSplitHandle
|
||
from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
|
||
csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode()
|
||
book=openpyxl.Workbook();sheet=book.active
|
||
for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row)
|
||
out=io.BytesIO();book.save(out)
|
||
for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]:
|
||
f=SimpleUploadedFile(name,data)
|
||
result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20)
|
||
chunks=result[0]['content'] if isinstance(result,list) else result['content']
|
||
self.assertGreater(len(chunks),1)
|
||
for a,b in zip(chunks,chunks[1:]):
|
||
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||
|
||
def test_xls_uses_row_overlap(self):
|
||
from common.handle.impl.text.xls_split_handle import handle_sheet
|
||
data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)]
|
||
sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i])
|
||
chunks=handle_sheet('test',sheet,120,20)['content']
|
||
for a,b in zip(chunks,chunks[1:]):
|
||
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||
|
||
def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self):
|
||
migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form')
|
||
model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock())
|
||
manager=Mock();manager.using.return_value.filter.return_value=[model]
|
||
apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager))
|
||
editor=SimpleNamespace(connection=SimpleNamespace(alias='default'))
|
||
migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor)
|
||
self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p'])
|
||
model.save.assert_called_once()
|
||
|
||
def test_generation_parameters_reach_stream_and_nonstream_http_body(self):
|
||
import json
|
||
import httpx
|
||
from openai import OpenAI
|
||
from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
|
||
from langchain_core.messages import HumanMessage
|
||
requests=[]
|
||
def transport(request):
|
||
body=json.loads(request.content);requests.append(body)
|
||
if body.get('stream'):
|
||
events=[]
|
||
for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]:
|
||
item={'id':'test','object':'chat.completion.chunk','model':'test','created':1,
|
||
'choices':[{'index':0,'delta':delta,'finish_reason':finish}]}
|
||
events.append('data: '+json.dumps(item)+'\n\n')
|
||
return httpx.Response(200,headers={'Content-Type':'text/event-stream'},
|
||
text=''.join(events)+'data: [DONE]\n\n')
|
||
return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1,
|
||
'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}],
|
||
'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}})
|
||
with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http:
|
||
client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http)
|
||
model=BaiLianChatModel.new_instance('LLM','test',{
|
||
'api_base':'http://example.invalid/v1','api_key':'test-only'},
|
||
temperature=0.23,top_p=0.42,max_tokens=10)
|
||
model.client=client.chat.completions
|
||
model.root_client=client
|
||
model.streaming=False
|
||
self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok')
|
||
model.streaming=True
|
||
self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok')
|
||
self.assertEqual(len(requests),2)
|
||
self.assertEqual([bool(r.get('stream')) for r in requests],[False,True])
|
||
for req in requests:
|
||
self.assertEqual(req['temperature'],0.23)
|
||
self.assertEqual(req['top_p'],0.42)
|
||
|
||
def test_new_knowledge_tasks_use_isolated_queue(self):
|
||
from ops import celery_app
|
||
import knowledge.tasks
|
||
for name in ('celery:embedding_by_document','celery:embedding_by_knowledge',
|
||
'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'):
|
||
self.assertIn(name,celery_app.tasks)
|
||
route=celery_app.amqp.router.route({},name)
|
||
self.assertEqual(route['queue'].name,'uniskb_knowledge')
|