UnisKB/apps/knowledge/test_knowledge_ext.py

294 lines
18 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Real isolated persistence and document parsing regression checks."""
import io
import importlib
import unittest
from types import SimpleNamespace
from unittest.mock import patch, Mock
from django.test import TestCase
from django.core.files.uploadedfile import SimpleUploadedFile
from common.exception.app_exception import AppApiException
from common.utils.split_model import smart_split_paragraph, SplitModel
from common.utils.table_split import split_table_rows
from knowledge.models import KnowledgeFolder, Knowledge
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules
class MetadataTests(TestCase):
def setUp(self):
KnowledgeFolder.objects.create(id='default', name='root')
self.k = Knowledge.objects.create(name='test', desc='', folder_id='default',
meta={'internal': {'preserve': True}})
self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester')
def test_metadata_persists_and_preserves_other_settings(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '})
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
self.assertEqual(self.k.meta['internal'], {'preserve': True})
result = Metadata.one('default', str(self.k.id))
self.assertEqual(len(result['fields']), 1)
self.assertEqual(result['updated_by'], '测试员')
self.assertTrue(result['updated_at'])
def test_partial_metadata_update_does_not_clear_other_fields(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'})
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'})
old = Metadata.one('default', str(self.k.id))
self.assertEqual(old['fields'][0]['name'], '业务领域')
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]})
self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')])
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码')
Metadata.update('default', str(self.k.id), self.user, {'fields': []})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], [])
self.k.refresh_from_db()
self.assertEqual(self.k.meta['internal'], {'preserve': True})
def test_custom_field_validation_does_not_write(self):
cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}],
[{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}],
[{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}],
[{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}],
[{'name': str(i), 'value': ''} for i in range(51)]]
for fields in cases:
with self.subTest(fields=fields), self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, {'fields': fields})
with self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'})
self.k.refresh_from_db()
self.assertNotIn('business_metadata', self.k.meta)
def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self):
from users.models import User
creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True)
original_time = self.k.create_time
result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
self.k.refresh_from_db()
self.assertEqual(self.k.user_id, creator.id)
self.assertEqual(self.k.create_time, original_time)
self.assertEqual(result['creator_name'], '新创建者')
self.assertEqual(result['user_id'], str(creator.id))
self.assertTrue(result['update_time'])
self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']])
def test_rejected_pending_disabled_creators_cannot_be_selected(self):
from users.models import User, UserActivationApplication
creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True)
application = UserActivationApplication.objects.create(user=creator, status='PENDING')
for status in ('PENDING', 'REJECTED'):
application.status = status; application.save()
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']])
application.status = 'APPROVED'; application.save()
creator.is_active = False; creator.save()
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)})
self.k.refresh_from_db()
self.assertIsNone(self.k.user_id)
self.assertNotIn('business_metadata', self.k.meta)
def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self):
from knowledge.serializers.knowledge import KnowledgeSerializer
other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={})
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
operation._validated_data = operation.initial_data
operation._errors = {}
params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'}
with patch.object(operation, 'is_valid'):
operation.edit({'embedding_params': params}, select_one=False)
operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False)
self.k.refresh_from_db(); other.refresh_from_db()
self.assertEqual(self.k.meta['embedding_params'], params)
self.assertNotIn('embedding_params', other.meta)
with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException):
operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False)
self.k.refresh_from_db()
self.assertEqual(self.k.meta['embedding_params'], params)
def test_general_settings_cannot_restore_stale_metadata(self):
from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]})
self.k.refresh_from_db()
protected = self.k.meta['business_metadata']
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
operation._validated_data = operation.initial_data
operation._errors = {}
with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'):
operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False)
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata'], protected)
self.assertEqual(self.k.meta['selector'], 'body')
self.assertNotIn('parser_id', self.k.meta)
def test_invalid_types_and_unknown_fields_do_not_write(self):
for data in ([1], {'business_domain': 123}, {'business_domain': False},
{'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}):
with self.subTest(data=data), self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, data)
self.k.refresh_from_db()
self.assertNotIn('business_metadata', self.k.meta)
def test_workspace_boundary(self):
for fn in (Metadata.one, Rules.one):
with self.assertRaises(AppApiException):
fn('other', str(self.k.id))
def test_segmentation_and_metadata_updates_preserve_each_other(self):
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20})
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20)
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维')
def test_segmentation_rejects_invalid_and_fractional_values(self):
for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10),
(100,1.2), (True,0), (100,False), ('100.5',0)):
with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException):
Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap})
def test_upload_inherits_defaults_and_accepts_explicit_override(self):
from knowledge.serializers.document import DocumentSerializers
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_size'], 500)
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
serializer.parse({'file':[SimpleUploadedFile('default.txt',b'test')]})
self.assertEqual(split.call_args.args[-2:], (500,0))
Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20})
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
f=SimpleUploadedFile('test.txt',b'test')
serializer.parse({'file':[f]})
self.assertEqual(split.call_args.args[-2:], (100,20))
serializer.parse({'file':[f],'limit':'200','overlap':'0'})
self.assertEqual(split.call_args.args[-2:], (200,0))
with self.assertRaises(AppApiException):
serializer.parse({'file':[f],'limit':'100','overlap':'100'})
class ParserTests(unittest.TestCase):
text=''.join(chr(0x4e00+i) for i in range(500))
def check_overlap(self, parts):
self.assertGreater(len(parts),1)
self.assertTrue(all(len(p)<=100 for p in parts))
self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1)))
def test_plain_text_overlap(self):
self.check_overlap(smart_split_paragraph(self.text,100,20))
def test_pdf_toc_and_internal_link_calls_pass_overlap(self):
from common.handle.impl.text.pdf_split_handle import PdfSplitHandle
doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1,
load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text))
result=PdfSplitHandle.handle_toc(doc,100,20)
self.check_overlap([p['content'] for p in result])
# 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。
import ast,inspect,textwrap
for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links):
calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method))))
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph']
self.assertTrue(calls)
self.assertTrue(all(len(n.args)==3 for n in calls))
def test_table_overlap_preserves_rows_headers_and_termination(self):
header='| id | value |\n| --- | --- |\n'
rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)]
chunks=split_table_rows(header,iter(rows),120,20)
self.assertGreater(len(chunks),1)
self.assertLess(len(chunks),len(rows))
self.assertTrue(all(c['content'].startswith(header) for c in chunks))
self.assertTrue(all(len(c['content'])<=120 for c in chunks))
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
for row in rows:self.assertTrue(any(row in c['content'] for c in chunks))
def test_oversized_table_row_is_not_truncated(self):
row='| '+'x'*500+' |\n'
result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20)
self.assertTrue(any(row in p['content'] for p in result))
self.assertIn('last',result[-1]['content'])
def test_csv_and_xlsx_use_row_overlap(self):
import openpyxl
from common.handle.impl.text.csv_split_handle import CsvSplitHandle
from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode()
book=openpyxl.Workbook();sheet=book.active
for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row)
out=io.BytesIO();book.save(out)
for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]:
f=SimpleUploadedFile(name,data)
result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20)
chunks=result[0]['content'] if isinstance(result,list) else result['content']
self.assertGreater(len(chunks),1)
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
def test_xls_uses_row_overlap(self):
from common.handle.impl.text.xls_split_handle import handle_sheet
data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)]
sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i])
chunks=handle_sheet('test',sheet,120,20)['content']
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self):
migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form')
model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock())
manager=Mock();manager.using.return_value.filter.return_value=[model]
apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager))
editor=SimpleNamespace(connection=SimpleNamespace(alias='default'))
migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor)
self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p'])
model.save.assert_called_once()
def test_generation_parameters_reach_stream_and_nonstream_http_body(self):
import json
import httpx
from openai import OpenAI
from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
from langchain_core.messages import HumanMessage
requests=[]
def transport(request):
body=json.loads(request.content);requests.append(body)
if body.get('stream'):
events=[]
for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]:
item={'id':'test','object':'chat.completion.chunk','model':'test','created':1,
'choices':[{'index':0,'delta':delta,'finish_reason':finish}]}
events.append('data: '+json.dumps(item)+'\n\n')
return httpx.Response(200,headers={'Content-Type':'text/event-stream'},
text=''.join(events)+'data: [DONE]\n\n')
return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1,
'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}],
'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}})
with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http:
client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http)
model=BaiLianChatModel.new_instance('LLM','test',{
'api_base':'http://example.invalid/v1','api_key':'test-only'},
temperature=0.23,top_p=0.42,max_tokens=10)
model.client=client.chat.completions
model.root_client=client
model.streaming=False
self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok')
model.streaming=True
self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok')
self.assertEqual(len(requests),2)
self.assertEqual([bool(r.get('stream')) for r in requests],[False,True])
for req in requests:
self.assertEqual(req['temperature'],0.23)
self.assertEqual(req['top_p'],0.42)
def test_new_knowledge_tasks_use_isolated_queue(self):
from ops import celery_app
import knowledge.tasks
for name in ('celery:embedding_by_document','celery:embedding_by_knowledge',
'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'):
self.assertIn(name,celery_app.tasks)
route=celery_app.amqp.router.route({},name)
self.assertEqual(route['queue'].name,'uniskb_knowledge')