83 lines
4.0 KiB
Python
83 lines
4.0 KiB
Python
"""Knowledge import uses the same authorized parser and durable task result as mindmaps."""
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
from rest_framework.exceptions import PermissionDenied, ValidationError
|
|
from knowledge.models import Knowledge
|
|
from system_manage import ai_hub as hub
|
|
|
|
|
|
@transaction.atomic
|
|
def parser_binding(workspace, knowledge_id, user, data=None):
|
|
knowledge = Knowledge.objects.select_for_update().filter(id=knowledge_id, workspace_id=workspace).first()
|
|
if not knowledge:
|
|
raise ValidationError('知识库不存在')
|
|
meta = dict(knowledge.meta or {})
|
|
if data is not None:
|
|
parser_id = data.get('parser_id')
|
|
if parser_id:
|
|
parser = hub.one('parsers', parser_id, workspace, user)
|
|
if not parser.enabled:
|
|
raise ValidationError('解析服务已停用')
|
|
meta['parser_id'] = str(parser_id) if parser_id else ''
|
|
knowledge.meta = meta
|
|
knowledge.save(update_fields=['meta', 'update_time'])
|
|
return {'parser_id': meta.get('parser_id', '')}
|
|
|
|
|
|
def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter, limit, overlap):
|
|
knowledge = Knowledge.objects.get(id=knowledge_id, workspace_id=workspace)
|
|
parser_id = (knowledge.meta or {}).get('parser_id')
|
|
if not parser_id:
|
|
return None
|
|
from system_manage.parsers import MINERU_INPUTS, extension
|
|
if extension(file.name) not in MINERU_INPUTS:
|
|
# Preserve existing knowledge converters, segmentation and attachment ownership.
|
|
return None
|
|
if user is None:
|
|
raise PermissionDenied('解析服务调用需要用户身份')
|
|
from system_manage.ai_hub_worker import execute
|
|
# Mark as running before commit: the asynchronous consumer cannot claim the same task.
|
|
with transaction.atomic():
|
|
task = hub.submit(workspace, user, {'name': file.name[:128], 'nodes': [
|
|
{'type': 'parse', 'resource_id': parser_id}, {'type': 'render'}]}, upload=file)
|
|
task.status = 'RUNNING'
|
|
task.started_at = timezone.now()
|
|
task.save(update_fields=['status', 'started_at'])
|
|
execute(task)
|
|
file.seek(0)
|
|
if task.status != 'SUCCESS':
|
|
raise ValidationError(task.error or '解析失败')
|
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
|
from common.handle.impl.text.text_split_handle import TextSplitHandle
|
|
content = knowledge_images(task.result['markdown'], workspace, knowledge_id, user).encode('utf-8')
|
|
converted = SimpleUploadedFile(file.name + '.md', content)
|
|
result = TextSplitHandle().handle(converted, patterns, with_filter, limit,
|
|
lambda _: content, lambda _: None, overlap)
|
|
result['name'] = file.name
|
|
return result, {'resource_id': str(parser_id), 'task_id': str(task.id),
|
|
'resource_version': task.snapshot['nodes'][0]['resource_version']}
|
|
|
|
|
|
def knowledge_images(markdown, workspace, knowledge_id, user):
|
|
"""Materialize parser images as knowledge attachments, with existing reader permissions and retention."""
|
|
import mimetypes
|
|
import uuid
|
|
from knowledge.models import File, FileSourceType
|
|
from system_manage.ai_hub_assets import ASSET, image_bytes
|
|
replacements = {}
|
|
for match in ASSET.finditer(markdown):
|
|
if match[0] in replacements:
|
|
continue
|
|
raw, meta = image_bytes(workspace, user, match[2])
|
|
file_id = uuid.uuid5(uuid.NAMESPACE_URL, f'knowledge-parser:{knowledge_id}:{match[2]}')
|
|
if not File.objects.filter(pk=file_id).exists():
|
|
suffix = mimetypes.guess_extension(meta['mime']) or '.jpg'
|
|
attachment = File(id=file_id, file_name='parsed-image' + suffix,
|
|
source_type=FileSourceType.KNOWLEDGE, source_id=str(knowledge_id),
|
|
meta={'knowledge_id': str(knowledge_id), 'parser_asset_id': match[2]})
|
|
attachment.save(raw)
|
|
replacements[match[0]] = './oss/file/' + str(file_id)
|
|
for before, after in replacements.items():
|
|
markdown = markdown.replace(before, after)
|
|
return markdown
|