UnisKB/apps/system_manage/ai_hub_knowledge.py

83 lines
4.0 KiB
Python

"""Knowledge import uses the same authorized parser and durable task result as mindmaps."""
from django.db import transaction
from django.utils import timezone
from rest_framework.exceptions import PermissionDenied, ValidationError
from knowledge.models import Knowledge
from system_manage import ai_hub as hub
@transaction.atomic
def parser_binding(workspace, knowledge_id, user, data=None):
knowledge = Knowledge.objects.select_for_update().filter(id=knowledge_id, workspace_id=workspace).first()
if not knowledge:
raise ValidationError('知识库不存在')
meta = dict(knowledge.meta or {})
if data is not None:
parser_id = data.get('parser_id')
if parser_id:
parser = hub.one('parsers', parser_id, workspace, user)
if not parser.enabled:
raise ValidationError('解析服务已停用')
meta['parser_id'] = str(parser_id) if parser_id else ''
knowledge.meta = meta
knowledge.save(update_fields=['meta', 'update_time'])
return {'parser_id': meta.get('parser_id', '')}
def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter, limit, overlap):
knowledge = Knowledge.objects.get(id=knowledge_id, workspace_id=workspace)
parser_id = (knowledge.meta or {}).get('parser_id')
if not parser_id:
return None
from system_manage.parsers import MINERU_INPUTS, extension
if extension(file.name) not in MINERU_INPUTS:
# Preserve existing knowledge converters, segmentation and attachment ownership.
return None
if user is None:
raise PermissionDenied('解析服务调用需要用户身份')
from system_manage.ai_hub_worker import execute
# Mark as running before commit: the asynchronous consumer cannot claim the same task.
with transaction.atomic():
task = hub.submit(workspace, user, {'name': file.name[:128], 'nodes': [
{'type': 'parse', 'resource_id': parser_id}, {'type': 'render'}]}, upload=file)
task.status = 'RUNNING'
task.started_at = timezone.now()
task.save(update_fields=['status', 'started_at'])
execute(task)
file.seek(0)
if task.status != 'SUCCESS':
raise ValidationError(task.error or '解析失败')
from django.core.files.uploadedfile import SimpleUploadedFile
from common.handle.impl.text.text_split_handle import TextSplitHandle
content = knowledge_images(task.result['markdown'], workspace, knowledge_id, user).encode('utf-8')
converted = SimpleUploadedFile(file.name + '.md', content)
result = TextSplitHandle().handle(converted, patterns, with_filter, limit,
lambda _: content, lambda _: None, overlap)
result['name'] = file.name
return result, {'resource_id': str(parser_id), 'task_id': str(task.id),
'resource_version': task.snapshot['nodes'][0]['resource_version']}
def knowledge_images(markdown, workspace, knowledge_id, user):
"""Materialize parser images as knowledge attachments, with existing reader permissions and retention."""
import mimetypes
import uuid
from knowledge.models import File, FileSourceType
from system_manage.ai_hub_assets import ASSET, image_bytes
replacements = {}
for match in ASSET.finditer(markdown):
if match[0] in replacements:
continue
raw, meta = image_bytes(workspace, user, match[2])
file_id = uuid.uuid5(uuid.NAMESPACE_URL, f'knowledge-parser:{knowledge_id}:{match[2]}')
if not File.objects.filter(pk=file_id).exists():
suffix = mimetypes.guess_extension(meta['mime']) or '.jpg'
attachment = File(id=file_id, file_name='parsed-image' + suffix,
source_type=FileSourceType.KNOWLEDGE, source_id=str(knowledge_id),
meta={'knowledge_id': str(knowledge_id), 'parser_asset_id': match[2]})
attachment.save(raw)
replacements[match[0]] = './oss/file/' + str(file_id)
for before, after in replacements.items():
markdown = markdown.replace(before, after)
return markdown