607 lines
39 KiB
Python
607 lines
39 KiB
Python
import json
|
|
from unittest import TestCase
|
|
from unittest.mock import patch, Mock
|
|
from django.db import transaction
|
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
|
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
|
|
from users.models import User
|
|
from users.models.user_manage_ext import UserActivationApplication
|
|
from system_manage import ai_hub as hub
|
|
from system_manage.ai_hub_worker import claim, execute
|
|
from system_manage.models.ai_hub import AIFlow, AIParserResource, AITask, AIArtifact, AIService
|
|
|
|
|
|
class NativeParserTests(TestCase):
|
|
@patch('requests.post')
|
|
def test_mineru_276_adapter_keeps_six_level_structure_and_images(self, post):
|
|
from types import SimpleNamespace
|
|
from system_manage.parsers.legacy import parse
|
|
markdown = '\n'.join('#' * level + ' Level' + str(level) for level in range(1, 7))
|
|
middle = {'pdf_info': [{'para_blocks': [{'type': 'title', 'level': 6}]}]}
|
|
images = {'figure.jpg': 'data:image/jpeg;base64,dGVzdA=='}
|
|
post.return_value = Mock(status_code=200)
|
|
post.return_value.json.return_value = {'version': '2.7.6', 'backend': 'pipeline', 'results': {
|
|
'sample': {'md_content': markdown, 'middle_json': json.dumps(middle), 'images': images}}}
|
|
result = parse(SimpleNamespace(id='test-parser', endpoint='http://parser.invalid'), 'sample.pdf', b'pdf',
|
|
{'engine_version': '2.7.6', 'backend': 'pipeline'}, {})
|
|
self.assertEqual(post.call_args.args[0], 'http://parser.invalid/file_parse')
|
|
self.assertEqual(post.call_args.kwargs['data']['return_middle_json'], 'true')
|
|
self.assertEqual(result['markdown'], markdown)
|
|
self.assertEqual(result['source']['middle_json'], middle)
|
|
self.assertEqual(result['images'], images)
|
|
node = hub.markdown_tree(markdown)
|
|
for level in range(1, 7):
|
|
node = node['children'][0]
|
|
self.assertEqual(node['text'], 'Level' + str(level))
|
|
|
|
def test_parser_catalogue_supports_the_271_baseline_and_existing_versions(self):
|
|
from system_manage.parsers.catalog import catalogue, validate_options
|
|
versions = {row['id']: row for row in catalogue()['versions']}
|
|
self.assertEqual(versions['2.7.1']['backends'], versions['2.7.4']['backends'])
|
|
for version in ('2.7.1', '2.7.4', '2.7.6', '3.4.5'):
|
|
options = validate_options({'engine_version': version, 'backend': 'pipeline'}, 'http://example.invalid:8000')
|
|
self.assertEqual(options['engine_version'], version)
|
|
self.assertEqual(options['protocol'], 'legacy')
|
|
|
|
def test_new_legacy_parser_defaults_to_271_without_changing_explicit_274(self):
|
|
from system_manage.parsers.catalog import validate_options
|
|
self.assertEqual(validate_options({}, 'http://example.invalid:8000')['engine_version'], '2.7.1')
|
|
self.assertEqual(validate_options({'engine_version': '2.7.4'}, 'http://example.invalid:8000')['engine_version'], '2.7.4')
|
|
|
|
def test_word_preserves_heading_only_sections_tables_and_images(self):
|
|
import io
|
|
from docx import Document
|
|
from PIL import Image
|
|
from system_manage.parsers.native import parse
|
|
|
|
document = Document()
|
|
document.add_heading('Chapter42', 1)
|
|
document.add_heading('Section42', 2)
|
|
document.add_heading('Detail42', 3)
|
|
document.add_paragraph('Body42')
|
|
table = document.add_table(rows=2, cols=2)
|
|
table.cell(0, 0).text = 'Name'
|
|
table.cell(0, 1).text = 'Value'
|
|
table.cell(1, 0).text = 'Count'
|
|
table.cell(1, 1).text = '42'
|
|
image = io.BytesIO()
|
|
Image.new('RGB', (2, 2), 'white').save(image, format='PNG')
|
|
image.seek(0)
|
|
document.add_picture(image)
|
|
buffer = io.BytesIO()
|
|
document.save(buffer)
|
|
|
|
with patch('common.handle.impl.text.doc_split_handle.DocSplitHandle.handle',
|
|
side_effect=AssertionError('Knowledge chunking must not run')):
|
|
markdown = parse('headings.docx', buffer.getvalue())['markdown']
|
|
self.assertEqual([line for line in markdown.splitlines() if line.startswith('#')],
|
|
['# Chapter42', '## Section42', '### Detail42'])
|
|
self.assertIn('Body42', markdown)
|
|
self.assertIn('| Count | 42 |', markdown)
|
|
self.assertIn('data:image/png;base64,', markdown)
|
|
self.assertNotIn('./oss/file/', markdown)
|
|
tree = hub.markdown_tree(markdown)
|
|
self.assertEqual(tree['children'][0]['children'][0]['children'][0]['text'], 'Detail42')
|
|
|
|
def test_html_and_htm_preserve_headings_lists_and_links_without_chunking(self):
|
|
from system_manage.parsers.native import parse
|
|
content = (b'<meta charset="utf-8"><h1>Chapter42</h1><h2>Section42</h2>'
|
|
b'<h3>Detail42</h3><p>Body42</p><ul><li>Parent42'
|
|
b'<ul><li>Child42</li></ul></li></ul>'
|
|
b'<p><a href="https://example.org">Link42</a></p>')
|
|
for suffix in ('html', 'htm'):
|
|
with self.subTest(suffix=suffix), patch(
|
|
'common.handle.impl.text.html_split_handle.HTMLSplitHandle.handle',
|
|
side_effect=AssertionError('Knowledge chunking must not run')):
|
|
markdown = parse('headings.' + suffix, content)['markdown']
|
|
self.assertEqual([line for line in markdown.splitlines() if line.startswith('#')],
|
|
['# Chapter42', '## Section42', '### Detail42'])
|
|
self.assertIn('Child42', markdown)
|
|
self.assertIn('[Link42](https://example.org)', markdown)
|
|
|
|
def test_xlsx_uses_cached_formula_values_and_does_not_evaluate_formulas(self):
|
|
import io
|
|
import zipfile
|
|
from openpyxl import Workbook
|
|
from system_manage.parsers.native import parse
|
|
|
|
workbook = Workbook()
|
|
workbook.active.append(['Cached', 'Uncached', 'Literal'])
|
|
workbook.active.append(['=6*7', '=2+3', 17])
|
|
buffer = io.BytesIO()
|
|
workbook.save(buffer)
|
|
cached = io.BytesIO()
|
|
with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as source, zipfile.ZipFile(cached, 'w') as target:
|
|
for item in source.infolist():
|
|
data = source.read(item.filename)
|
|
if item.filename == 'xl/worksheets/sheet1.xml':
|
|
self.assertIn(b'<f>6*7</f><v></v>', data)
|
|
data = data.replace(b'<f>6*7</f><v></v>', b'<f>6*7</f><v>42</v>')
|
|
target.writestr(item, data)
|
|
markdown = parse('cached.xlsx', cached.getvalue())['markdown']
|
|
self.assertIn('| 42 | | 17 |', markdown)
|
|
self.assertNotIn('=6*7', markdown)
|
|
self.assertNotIn('=2+3', markdown)
|
|
|
|
|
|
class HubTests(TestCase):
|
|
def setUp(self):
|
|
import tempfile
|
|
from django.test import override_settings
|
|
from pathlib import Path
|
|
self.asset_temp = tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[3] / '.codex_tmp')
|
|
self.addCleanup(self.asset_temp.cleanup)
|
|
self.asset_settings = override_settings(AI_HUB_ASSET_DIR=self.asset_temp.name)
|
|
self.asset_settings.enable()
|
|
self.addCleanup(self.asset_settings.disable)
|
|
self.atomic = transaction.atomic()
|
|
self.atomic.__enter__()
|
|
self.admin = User.objects.create(username='admin', nick_name='admin', role='ADMIN', is_active=True)
|
|
self.user = User.objects.create(username='user', nick_name='user', role='USER', is_active=True)
|
|
self.other = User.objects.create(username='other', nick_name='other', role='USER', is_active=True)
|
|
|
|
def tearDown(self):
|
|
transaction.set_rollback(True)
|
|
self.atomic.__exit__(None, None, None)
|
|
|
|
def parser(self, **kwargs):
|
|
return AIParserResource.objects.create(workspace_id='default', owner_id=self.admin.id,
|
|
name='测试解析', endpoint='http://parser.test/api', allowed_users=[str(self.user.id)], **kwargs)
|
|
|
|
def test_parser_grant_and_workspace_isolation(self):
|
|
p = self.parser()
|
|
self.assertEqual(hub.one('parsers', p.id, 'default', self.user).id, p.id)
|
|
with self.assertRaises(NotFound): hub.one('parsers', p.id, 'default', self.other)
|
|
with self.assertRaises(NotFound): hub.one('parsers', p.id, 'other', self.admin)
|
|
with self.assertRaises(PermissionDenied): hub.one('parsers', p.id, 'default', self.user, edit=True)
|
|
|
|
def test_only_admin_can_register_parser(self):
|
|
with self.assertRaises(PermissionDenied): hub.parser_save('default', self.user, {})
|
|
|
|
def test_authorization_excludes_rejected_pending_disabled(self):
|
|
application = UserActivationApplication.objects.create(user=self.user, status='REJECTED')
|
|
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
|
|
application.status = 'PENDING'; application.save()
|
|
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
|
|
application.status = 'APPROVED'; application.save()
|
|
self.assertEqual(hub.users([str(self.user.id)]), [str(self.user.id)])
|
|
self.user.is_active = False; self.user.save()
|
|
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
|
|
|
|
def test_parser_secret_never_serialized_and_preserved(self):
|
|
with patch('common.utils.rsa_util.rsa_long_encrypt', return_value='encrypted'):
|
|
p = hub.parser_save('default', self.admin, {'name': '解析', 'endpoint': 'http://parser.test/api', 'api_key': 'private', 'allowed_users': []})
|
|
self.assertEqual(p.credential, 'encrypted')
|
|
self.assertNotIn('private', json.dumps(hub.serialize(p)))
|
|
self.assertNotIn('encrypted', json.dumps(hub.serialize(p)))
|
|
p = hub.parser_save('default', self.admin, {'name': '解析2'}, p)
|
|
self.assertEqual(p.credential, 'encrypted')
|
|
|
|
def test_parser_url_and_options_validation(self):
|
|
for endpoint in ('file:///tmp/a', 'http://user:secret@server/api', 'http://server/api?key=secret'):
|
|
with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': endpoint})
|
|
with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/api', 'options': {'server_url': 'http://other'}})
|
|
|
|
def test_parser_toggle_preserves_shared_options_and_grants(self):
|
|
p = hub.parser_save('default', self.admin, {'name': 'shared', 'endpoint': 'http://server/v1',
|
|
'allowed_users': [str(self.user.id)], 'options': {'protocol': 'v1', 'tier': 'basic'}})
|
|
self.user.is_active = False
|
|
self.user.save(update_fields=['is_active'])
|
|
hub.parser_save('default', self.admin, {'enabled': False}, p)
|
|
p.refresh_from_db()
|
|
self.assertFalse(p.enabled)
|
|
self.assertEqual(p.allowed_users, [str(self.user.id)])
|
|
self.assertEqual(p.options, {'protocol': 'v1', 'tier': 'basic'})
|
|
self.assertEqual(p.name, 'shared')
|
|
|
|
@patch('requests.get')
|
|
def test_parser_connection_checks_protocol_and_does_not_return_service_content(self, get):
|
|
p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}})
|
|
get.return_value = Mock(status_code=200)
|
|
get.return_value.json.return_value = {'paths': {'/v1/tasks': {'post': {}}}, 'secret': 'not-for-browser'}
|
|
result = hub.parser_test(p)
|
|
self.assertTrue(result['connected'])
|
|
self.assertNotIn('secret', result)
|
|
self.assertEqual(get.call_args.args[0], 'http://server/openapi.json')
|
|
self.assertFalse(get.call_args.kwargs['allow_redirects'])
|
|
get.return_value.json.return_value = {'paths': {'/v1/tasks': {'get': {}}}}
|
|
with self.assertRaises(ValidationError): hub.parser_test(p)
|
|
p.endpoint = 'http://server/api'; p.options = {'protocol': 'legacy'}
|
|
get.return_value.json.return_value = {'paths': {'/file_parse': {'post': {}}}}
|
|
self.assertTrue(hub.parser_test(p)['connected'])
|
|
|
|
@patch('requests.get')
|
|
def test_parser_connection_failures_and_permissions(self, get):
|
|
import requests
|
|
from types import SimpleNamespace
|
|
from system_manage.views.ai_hub import ParserTest
|
|
p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}})
|
|
with self.assertRaises(PermissionDenied): ParserTest().post(SimpleNamespace(user=self.user), 'default', p.id)
|
|
get.assert_not_called()
|
|
get.return_value = Mock(status_code=302)
|
|
with self.assertRaises(ValidationError): hub.parser_test(p)
|
|
get.return_value = Mock(status_code=200)
|
|
get.return_value.json.side_effect = ValueError('sensitive server response')
|
|
with self.assertRaises(ValidationError) as error: hub.parser_test(p)
|
|
self.assertNotIn('sensitive', str(error.exception))
|
|
get.side_effect = requests.Timeout('sensitive URL')
|
|
with self.assertRaises(ValidationError) as error: hub.parser_test(p)
|
|
self.assertNotIn('sensitive', str(error.exception))
|
|
|
|
def test_flow_versions_are_immutable(self):
|
|
a = hub.flow_create('default', self.user, {'name': '文档', 'nodes': [{'type': 'render'}]})
|
|
b = hub.flow_create('default', self.user, {'name': '文档', 'previous_id': str(a.id), 'nodes': [{'type': 'render'}]})
|
|
self.assertEqual((a.revision, b.revision), (1, 2))
|
|
self.assertNotEqual(a.id, b.id)
|
|
|
|
def test_invalid_flow_and_ungranted_parser_rejected(self):
|
|
p = self.parser()
|
|
for nodes in ([{'type': 'render'}, {'type': 'parse', 'resource_id': str(p.id)}], [{'type': 'custom'}], [{'type': 'render'}, {'type': 'render'}]):
|
|
with self.assertRaises(ValidationError): hub.validate_nodes('default', self.user, nodes)
|
|
with self.assertRaises(NotFound): hub.validate_nodes('default', self.other, [{'type': 'parse', 'resource_id': str(p.id)}, {'type': 'render'}])
|
|
|
|
def test_submit_claim_and_rule_render(self):
|
|
task = hub.submit('default', self.user, {'markdown': '# 标题\n## 内容\n事实42\n```\n# 不作标题\n```', 'nodes': [{'type': 'render'}]})
|
|
picked = claim()
|
|
self.assertEqual(picked.id, task.id)
|
|
self.assertIsNone(claim())
|
|
execute(picked); picked.refresh_from_db()
|
|
self.assertEqual(picked.status, 'SUCCESS')
|
|
tree = picked.result['tree']
|
|
self.assertEqual(tree['children'][0]['text'], '标题')
|
|
self.assertNotIn('不作标题', json.dumps(tree, ensure_ascii=False))
|
|
|
|
def test_parser_task_preserves_original_and_source(self):
|
|
parser = self.parser()
|
|
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('scan.pdf', b'pdf'))
|
|
response = Mock(status_code=200)
|
|
response.json.return_value = {'results': {'scan': {'md_content': '# 内容\n数值42', 'middle_json': {'pages': [1]}}}}
|
|
with patch('requests.post', return_value=response) as post:
|
|
execute(claim())
|
|
self.assertFalse(post.call_args.kwargs['allow_redirects'])
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, 'SUCCESS')
|
|
self.assertEqual(task.result['original'], '# 内容\n数值42')
|
|
self.assertEqual(task.result['source']['middle_json']['pages'], [1])
|
|
|
|
def test_builtin_docx_works_without_external_parser(self):
|
|
import io
|
|
from docx import Document as WordDocument
|
|
document = WordDocument()
|
|
document.add_heading('Native42', 0)
|
|
document.add_paragraph('保留原文数值42')
|
|
buffer = io.BytesIO(); document.save(buffer)
|
|
task = hub.submit('default', self.user, {'nodes': [
|
|
{'type': 'parse', 'processor': 'builtin'}, {'type': 'render'}]},
|
|
SimpleUploadedFile('native.docx', buffer.getvalue()))
|
|
with patch('requests.post') as external:
|
|
execute(claim())
|
|
external.assert_not_called()
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, 'SUCCESS', task.error)
|
|
self.assertIn('42', task.result['markdown'])
|
|
self.assertEqual(task.result['source']['engine'], 'maxkb-native')
|
|
|
|
def test_parser_disabled_or_changed_after_submit_fails(self):
|
|
parser = self.parser()
|
|
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
|
|
parser.enabled = False; parser.save()
|
|
with patch('requests.post') as post: execute(claim()); post.assert_not_called()
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, 'FAILED')
|
|
|
|
def test_model_calls_existing_provider_with_selected_parameters(self):
|
|
from models_provider.models import Model
|
|
model = Model.objects.create(name='整理', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
|
|
task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id), 'temperature': 0}, {'type': 'render'}]})
|
|
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿')
|
|
with patch('models_provider.tools.get_model', return_value=fake) as get:
|
|
execute(claim())
|
|
self.assertEqual(get.call_args.kwargs['temperature'], 0)
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.result['original'], '# 原稿')
|
|
self.assertEqual(task.result['markdown'], '# 整理稿')
|
|
|
|
def test_model_params_form_defaults_win_over_absent_node_values(self):
|
|
from models_provider.models import Model
|
|
model = Model.objects.create(name='默认参数', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS',
|
|
model_params_form=[{'field': 'temperature', 'default_value': 0.7}, {'field': 'max_tokens', 'default_value': 8192}])
|
|
task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
|
|
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿')
|
|
with patch('models_provider.tools.get_model', return_value=fake) as get:
|
|
execute(claim())
|
|
self.assertEqual(get.call_args.kwargs['temperature'], 0.7)
|
|
self.assertEqual(get.call_args.kwargs['max_tokens'], 8192)
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, 'SUCCESS')
|
|
|
|
def test_provider_exception_is_sanitized(self):
|
|
parser = self.parser()
|
|
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
|
|
with patch('requests.post', side_effect=RuntimeError('private-url-key')): execute(claim())
|
|
task.refresh_from_db(); self.assertEqual(task.status, 'FAILED')
|
|
self.assertNotIn('private', task.error)
|
|
|
|
def test_empty_parse_does_not_succeed(self):
|
|
parser = self.parser()
|
|
response = Mock(status_code=200); response.json.return_value = {'results': {'doc': {'md_content': ''}}}
|
|
with patch('requests.post', return_value=response):
|
|
with self.assertRaises(ValidationError): hub.parse_file(parser, 'x.pdf', b'pdf')
|
|
|
|
def test_task_validation_error_is_displayed_without_framework_details(self):
|
|
parser = self.parser()
|
|
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
|
|
with patch('system_manage.ai_hub.parse_file', side_effect=ValidationError('解析失败,请检查文档')):
|
|
execute(claim())
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.error, '解析失败,请检查文档')
|
|
|
|
def test_artifact_review_preserves_original_and_detects_conflicts(self):
|
|
a = hub.artifact_save('default', self.user, {'name': '图谱', 'original': '# OCR原值', 'markdown': '# 复核结果'})
|
|
a = hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'original': '篡改原稿', 'markdown': '# 第二版'}, a)
|
|
self.assertEqual(a.original, '# OCR原值'); self.assertEqual(a.revision, 2)
|
|
self.assertEqual(a.history[0]['markdown'], '# 复核结果')
|
|
with self.assertRaises(ValidationError): hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'markdown': '# 旧提交'}, a)
|
|
with self.assertRaises(NotFound): hub.one('artifacts', a.id, 'default', self.other)
|
|
|
|
def test_published_service_is_version_pinned_and_disabled_stops_submit(self):
|
|
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
|
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
|
task = hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
|
self.assertEqual(task.snapshot['flow_revision'], 1)
|
|
service.enabled = False; service.save()
|
|
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
|
execute(claim()); task.refresh_from_db(); self.assertEqual(task.status, 'FAILED')
|
|
|
|
def test_unknown_ids_and_missing_inputs_are_validation_errors(self):
|
|
with self.assertRaises(ValidationError): hub.one('parsers', None, 'default', self.admin)
|
|
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'nodes': [{'type': 'render'}]})
|
|
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': 'x', 'nodes': [{'type': 'parse', 'resource_id': str(self.parser().id)}, {'type': 'render'}]})
|
|
|
|
def api(self, method, kind, data=None, pk=None, user=None, headers=None):
|
|
from rest_framework.test import APIRequestFactory, force_authenticate
|
|
from system_manage.views.ai_hub import HubView, PublishedRun
|
|
request = getattr(APIRequestFactory(), method)('/', data or {}, format='json', **(headers or {}))
|
|
force_authenticate(request, user=user or self.admin)
|
|
kwargs = {'workspace_id': 'default'}
|
|
if kind != 'published': kwargs['kind'] = kind
|
|
if pk: kwargs['pk'] = pk
|
|
view = PublishedRun if kind == 'published' else HubView
|
|
with patch('users.serializers.user.get_workspace_list_by_user', return_value=[{'id': 'default'}]):
|
|
response = view.as_view()(request, **kwargs)
|
|
return json.loads(response.content) if hasattr(response, 'content') else response.data
|
|
|
|
def test_api_service_key_rotation_revocation_and_real_task(self):
|
|
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
|
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
|
rotated = self.api('post', 'services', {'rotate_key': True}, service.id)['data']
|
|
self.assertTrue(rotated['api_key'])
|
|
self.assertNotIn('api_key', self.api('get', 'services', pk=service.id)['data'])
|
|
denied = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user)
|
|
self.assertNotEqual(denied['code'], 200)
|
|
accepted = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user, {'HTTP_X_SERVICE_KEY': rotated['api_key']})
|
|
self.assertEqual(accepted['code'], 200)
|
|
self.assertEqual(AITask.objects.get(id=accepted['data']['id']).status, 'PENDING')
|
|
self.api('post', 'services', {'revoke_key': True}, service.id)
|
|
service.refresh_from_db(); self.assertEqual(service.key_hash, '')
|
|
|
|
def test_api_cannot_edit_shared_service_and_stops_disabled_service(self):
|
|
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
|
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
|
response = self.api('post', 'services', {'enabled': False}, service.id, self.user)
|
|
self.assertNotEqual(response['code'], 200)
|
|
self.api('post', 'services', {'enabled': False}, service.id)
|
|
response = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user)
|
|
self.assertNotEqual(response['code'], 200)
|
|
|
|
def test_api_retry_resets_failure_without_duplicate_task(self):
|
|
task = hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]})
|
|
task.status = 'FAILED'; task.error = '中断'; task.save()
|
|
response = self.api('post', 'tasks', {}, task.id, self.user)
|
|
self.assertEqual(response['code'], 200)
|
|
task.refresh_from_db(); self.assertEqual(task.attempt, 2); self.assertEqual(task.status, 'PENDING')
|
|
self.assertEqual(AITask.objects.count(), 1)
|
|
response = self.api('post', 'tasks', {}, task.id, self.user)
|
|
self.assertNotEqual(response['code'], 200)
|
|
|
|
def test_knowledge_binding_and_shared_parser_execution(self):
|
|
from knowledge.models import KnowledgeFolder, Knowledge
|
|
from system_manage.ai_hub_knowledge import parser_binding, split_with_parser
|
|
KnowledgeFolder.objects.create(id='default', name='root')
|
|
knowledge = Knowledge.objects.create(name='测试知识库', desc='', folder_id='default', meta={'segmentation': {'chunk_size': 100}})
|
|
parser = self.parser()
|
|
parser_binding('default', knowledge.id, self.user, {'parser_id': str(parser.id)})
|
|
knowledge.refresh_from_db(); self.assertEqual(knowledge.meta['segmentation']['chunk_size'], 100)
|
|
with self.assertRaises(NotFound): parser_binding('default', knowledge.id, self.other, {'parser_id': str(parser.id)})
|
|
response = Mock(status_code=200)
|
|
response.json.return_value = {'results': {'scan': {'md_content': '# 章节\n识别结果42', 'middle_json': {'pages': [1]}}}}
|
|
file = SimpleUploadedFile('scan.pdf', b'pdf')
|
|
with patch('requests.post', return_value=response):
|
|
result, provenance = split_with_parser('default', knowledge.id, self.user, file, None, False, 100, 20)
|
|
task = AITask.objects.get(id=provenance['task_id'])
|
|
self.assertEqual(task.status, 'SUCCESS'); self.assertEqual(provenance['resource_id'], str(parser.id))
|
|
self.assertIn('识别结果42', json.dumps(result, ensure_ascii=False)); self.assertEqual(file.tell(), 0)
|
|
parser_binding('default', knowledge.id, self.user, {'parser_id': ''})
|
|
self.assertIsNone(split_with_parser('default', knowledge.id, None, file, None, False, 100, 0))
|
|
|
|
def test_model_selection_follows_view_grants_instead_of_ownership(self):
|
|
from models_provider.models import Model
|
|
from types import SimpleNamespace
|
|
model = Model.objects.create(name='私有', model_type='LLM', model_name='demo', provider='test', user_id=self.user.id, credential='', status='SUCCESS', model_params_form=[])
|
|
with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[]):
|
|
with self.assertRaises(PermissionDenied): hub.model_for('default', self.user, model.id)
|
|
grant = SimpleNamespace(target=str(model.id), permission_list=['VIEW'])
|
|
with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[grant]):
|
|
self.assertEqual(hub.model_for('default', self.user, model.id).id, model.id)
|
|
|
|
def test_queue_limit_applies_to_published_service_too(self):
|
|
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
|
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
|
for _ in range(10):
|
|
hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
|
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
|
self.assertEqual(AITask.objects.count(), 10)
|
|
|
|
def test_stopped_user_cannot_submit_directly(self):
|
|
self.user.is_active = False; self.user.save()
|
|
with self.assertRaises(PermissionDenied): hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]})
|
|
|
|
def test_v1_adapter_polls_registered_endpoint_and_preserves_source(self):
|
|
parser = self.parser(options={'protocol': 'v1', 'tier': 'flash'})
|
|
created = Mock(status_code=202); created.json.return_value = {'task_id': 'task_test', 'status': 'queued', 'result_url': 'http://untrusted/secret'}
|
|
status = Mock(status_code=200); status.json.return_value = {'task_id': 'task_test', 'status': 'completed'}
|
|
result = Mock(status_code=200); result.json.return_value = {'files': [{'status': 'completed', 'parse': {'engine': 'native'}, 'content': {'markdown': '# 识别结果42', 'middle_json': {'pages': [1]}}}]}
|
|
with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=created) as post, patch('system_manage.ai_hub_parser_v1.requests.get', side_effect=[status, result]) as get, patch('system_manage.ai_hub_parser_v1.time.sleep'):
|
|
parsed = hub.parse_file(parser, 'test.pdf', b'pdf')
|
|
self.assertEqual(parsed['markdown'], '# 识别结果42')
|
|
self.assertEqual(parsed['source']['middle_json']['pages'], [1])
|
|
self.assertEqual(get.call_args.args[0], parser.endpoint + '/tasks/task_test/result')
|
|
self.assertIn(('tier', 'flash'), post.call_args.kwargs['params'])
|
|
|
|
def test_v1_adapter_rejects_external_task_paths_and_failed_files(self):
|
|
parser = self.parser(options={'protocol': 'v1'})
|
|
response = Mock(status_code=202); response.json.return_value = {'task_id': '../../other', 'status': 'completed'}
|
|
with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=response), patch('system_manage.ai_hub_parser_v1.requests.get') as get:
|
|
with self.assertRaises(ValidationError): hub.parse_file(parser, 'test.pdf', b'pdf')
|
|
get.assert_not_called()
|
|
|
|
def image_markdown(self):
|
|
import base64
|
|
import io
|
|
from PIL import Image
|
|
content = io.BytesIO()
|
|
Image.new('RGB', (40, 20), '#2468ac').save(content, 'PNG')
|
|
return '# 图片验收\n).decode() + ')', content.getvalue()
|
|
|
|
def test_asset_submission_persistence_deduplication_and_isolation(self):
|
|
from system_manage.ai_hub_assets import ASSET, image_bytes, root
|
|
markdown, content = self.image_markdown()
|
|
task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
|
|
url = ASSET.search(task.input_data['markdown'])
|
|
self.assertIsNotNone(url)
|
|
self.assertNotIn('base64', task.input_data['markdown'])
|
|
self.assertEqual(image_bytes('default', self.user, url[2])[0], content)
|
|
with self.assertRaises(NotFound): image_bytes('default', self.other, url[2])
|
|
with self.assertRaises(NotFound): image_bytes('other', self.admin, url[2])
|
|
with self.assertRaises(NotFound): hub.submit('default', self.other, {'markdown': task.input_data['markdown'], 'nodes': [{'type': 'render'}]})
|
|
hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
|
|
self.assertEqual(len(list(root().glob('*.bin'))), 1)
|
|
|
|
def test_asset_cleanup_preserves_original_history_and_inflight(self):
|
|
import time
|
|
from system_manage.ai_hub_assets import ASSET, cleanup, root
|
|
markdown, _ = self.image_markdown()
|
|
artifact = hub.artifact_save('default', self.user, {'name': '测试图片', 'markdown': markdown, 'original': '', 'layout': 'tree'})
|
|
asset_id = ASSET.search(artifact.markdown)[2]
|
|
hub.artifact_save('default', self.user, {'name': artifact.name, 'revision': artifact.revision, 'markdown': '# 删除当前引用'}, artifact)
|
|
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0)
|
|
self.assertTrue((root() / (asset_id + '.bin')).exists())
|
|
# Only remove the isolated test record; existing shared records are never involved.
|
|
artifact.delete()
|
|
task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
|
|
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0)
|
|
execute(task)
|
|
self.assertEqual(cleanup(now=time.time() + 8 * 86400), 1)
|
|
self.assertTrue((root() / (asset_id + '.bin')).exists())
|
|
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 1)
|
|
self.assertFalse((root() / (asset_id + '.bin')).exists())
|
|
|
|
def test_asset_parser_relative_images_and_invalid_payload(self):
|
|
from system_manage.ai_hub_assets import asset_lock, normalize, ASSET
|
|
markdown, _ = self.image_markdown()
|
|
uri = markdown.split('](', 1)[1][:-1]
|
|
with asset_lock():
|
|
normalized = normalize('\n<img src="a.png">', 'default', self.user, {'a.png': uri})
|
|
self.assertEqual(len(ASSET.findall(normalized)), 2)
|
|
with self.assertRaises(ValidationError): normalize('', 'default', self.user)
|
|
|
|
def test_asset_bundle_is_offline_and_model_cannot_drop_images(self):
|
|
import io
|
|
import zipfile
|
|
from types import SimpleNamespace
|
|
from system_manage.views.ai_hub import MarkdownExport
|
|
from system_manage.ai_hub_assets import protect_images, restore_images
|
|
markdown, content = self.image_markdown()
|
|
artifact = hub.artifact_save('default', self.user, {'name': '测试导出', 'markdown': markdown})
|
|
response = MarkdownExport().post(SimpleNamespace(user=self.user, data={'markdown': artifact.markdown}), 'default')
|
|
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
|
self.assertEqual(len(archive.namelist()), 2)
|
|
exported = archive.read('document.md').decode()
|
|
self.assertNotIn('/admin/api/', exported)
|
|
self.assertIn('images/', exported)
|
|
self.assertEqual(archive.read(next(n for n in archive.namelist() if n.startswith('images/'))), content)
|
|
protected, images = protect_images(artifact.markdown)
|
|
self.assertNotIn('/admin/api/', protected)
|
|
self.assertEqual(restore_images(protected, images), artifact.markdown)
|
|
self.assertIn(images[0], restore_images('# 整理完成但遗漏图片', images))
|
|
self.assertEqual(restore_images(protected + protected, images).count(images[0]), 1)
|
|
self.assertIn(images[0], restore_images('AIHUBIMAGE 0001 TOKEN', images))
|
|
|
|
def test_organize_restores_eight_omitted_images_and_preserves_original(self):
|
|
from models_provider.models import Model
|
|
markdown, _ = self.image_markdown()
|
|
image = markdown.split('\n', 1)[1]
|
|
original = '# 报告\n' + '\n\n'.join(image for _ in range(8))
|
|
model = Model.objects.create(name='测试整理', model_type='LLM', model_name='demo', provider='test',
|
|
user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
|
|
task = hub.submit('default', self.admin, {'markdown': original, 'nodes': [
|
|
{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
|
|
normalized = task.input_data['markdown']
|
|
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理结果\n- 结论42')
|
|
with patch('models_provider.tools.get_model', return_value=fake):
|
|
execute(claim())
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, 'SUCCESS', task.error)
|
|
self.assertEqual(task.result['original'], normalized)
|
|
self.assertEqual(task.result['markdown'].count(', 8)
|
|
self.assertNotIn('AIHUBIMAGE', task.result['markdown'])
|
|
self.assertIn('结论42', task.result['markdown'])
|
|
self.assertNotIn('/admin/api/', fake.invoke.call_args.args[0][1].content)
|
|
|
|
def test_failed_organize_keeps_source_available(self):
|
|
from models_provider.models import Model
|
|
model = Model.objects.create(name='测试失败', model_type='LLM', model_name='demo', provider='test',
|
|
user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
|
|
task = hub.submit('default', self.admin, {'markdown': '# 保留原稿42', 'nodes': [
|
|
{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
|
|
with patch('models_provider.tools.get_model', side_effect=RuntimeError('private')): execute(claim())
|
|
task.refresh_from_db()
|
|
self.assertEqual(task.status, 'FAILED')
|
|
self.assertEqual(task.result['original'], '# 保留原稿42')
|
|
self.assertNotIn('private', task.error)
|
|
|
|
def test_single_file_progress_is_indeterminate_and_counts_are_real(self):
|
|
from system_manage.ai_hub_progress import parser_progress, task_progress
|
|
single = parser_progress({'status': 'running', 'progress': {'completed': 0, 'total': 1}})
|
|
self.assertIsNone(single['percent'])
|
|
self.assertEqual(single['stage'], '正在解析文档')
|
|
self.assertEqual(parser_progress({'status': 'queued'})['stage'], '等待解析服务处理')
|
|
self.assertEqual(parser_progress({'status': 'running', 'progress': {'completed': 2, 'total': 4}})['percent'], 50)
|
|
task = hub.submit('default', self.user, {'markdown': '# 测试', 'nodes': [{'type': 'render'}]})
|
|
self.assertIsNone(task_progress(task)['percent'])
|
|
task.status = 'RUNNING'; task.steps = [{'type': 'parse', 'status': 'RUNNING', 'progress': single}]
|
|
self.assertEqual(hub.serialize(task, True)['progress']['stage'], '正在解析文档')
|
|
self.assertIsNone(hub.serialize(task, True)['progress']['percent'])
|
|
|
|
def test_v1_progress_reaches_database_before_task_finishes(self):
|
|
parser = self.parser(options={'protocol': 'v1'})
|
|
task = hub.submit('default', self.user, {'nodes': [
|
|
{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('test.pdf', b'pdf'))
|
|
created = Mock(status_code=202); created.json.return_value = {'task_id': 'test_progress', 'status': 'queued'}
|
|
running = Mock(status_code=200); running.json.return_value = {'status': 'running', 'progress': {'completed': 0, 'total': 1}}
|
|
done = Mock(status_code=200); done.json.return_value = {'status': 'completed'}
|
|
result = Mock(status_code=200); result.json.return_value = {'files': [
|
|
{'status': 'completed', 'content': {'markdown': '# 完成42'}}]}
|
|
seen = []
|
|
responses = iter([running, done, result])
|
|
def get(*args, **kwargs):
|
|
row = AITask.objects.get(pk=task.pk)
|
|
seen.append(hub.serialize(row, True)['progress'])
|
|
return next(responses)
|
|
with patch('requests.post', return_value=created), patch('requests.get', side_effect=get), patch('system_manage.ai_hub_parser_v1.time.sleep'):
|
|
execute(claim())
|
|
self.assertEqual([item['stage'] for item in seen], ['等待解析服务处理', '正在解析文档', '获取解析结果'])
|
|
self.assertTrue(all(item['percent'] is None for item in seen))
|
|
task.refresh_from_db()
|
|
self.assertEqual(hub.serialize(task, True)['progress']['percent'], 100)
|