UnisKB/apps/system_manage/parsers/native.py

103 lines
5.3 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Reuse MaxKB's built-in converters; MinerU receives only supported visual documents."""
import base64
import io
import mimetypes
from pathlib import Path
import shutil
import subprocess
import tempfile
import zipfile
from rest_framework.exceptions import ValidationError
def parse(name, content):
from django.core.files.uploadedfile import SimpleUploadedFile
from system_manage.ai_hub import text
from . import extension
suffix = extension(name)
if suffix in ('md', 'txt'):
# Preserve headings and whitespace even when a document has no body paragraph.
try:
markdown = content.decode('utf-8-sig')
except UnicodeDecodeError:
from charset_normalizer import from_bytes
decoded = from_bytes(content).best()
if decoded is None:
raise ValidationError('无法识别文本编码,请使用 UTF-8')
markdown = str(decoded)
return {'markdown': text(markdown, '解析结果'), 'images': {},
'source': {'engine': 'maxkb-native', 'format': suffix}}
if suffix == 'doc':
command = shutil.which('libreoffice') or shutil.which('soffice')
if command is None:
raise ValidationError('DOC 格式需要 LibreOffice,请先另存为 DOCX')
with tempfile.TemporaryDirectory(prefix='maxkb-office-') as directory:
root = Path(directory); source = root / 'input.doc'; source.write_bytes(content)
try:
result = subprocess.run([command, '-env:UserInstallation=' + (root / 'profile').as_uri(),
'--headless', '--convert-to', 'docx', '--outdir', directory, str(source)],
capture_output=True, timeout=120)
except (OSError, subprocess.TimeoutExpired) as exc:
raise ValidationError('DOC 转换失败,请另存为 DOCX') from exc
output = root / 'input.docx'
if result.returncode or not output.is_file():
raise ValidationError('DOC 转换失败,请另存为 DOCX')
parsed = parse(Path(name).stem + '.docx', output.read_bytes())
parsed['source']['converted_from'] = 'doc'
return parsed
if suffix in ('docx', 'xlsx'):
try:
with zipfile.ZipFile(io.BytesIO(content)) as archive:
if sum(item.file_size for item in archive.infolist()) > 200 * 1024 * 1024:
raise ValidationError('文档解压后超过大小限制')
except zipfile.BadZipFile as exc:
raise ValidationError('文件格式无效') from exc
converters = {
'pdf': ('pdf_split_handle', 'PdfSplitHandle'),
'docx': ('doc_split_handle', 'DocSplitHandle'), 'xlsx': ('xlsx_split_handle', 'XlsxSplitHandle'),
'xls': ('xls_split_handle', 'XlsSplitHandle'), 'csv': ('csv_split_handle', 'CsvSplitHandle'),
'html': ('html_split_handle', 'HTMLSplitHandle'), 'htm': ('html_split_handle', 'HTMLSplitHandle'),
'md': ('text_split_handle', 'TextSplitHandle'), 'txt': ('text_split_handle', 'TextSplitHandle'),
}
if suffix not in converters:
raise ValidationError('内置处理器不支持该文件格式')
import importlib
module, converter = converters[suffix]
handler = getattr(importlib.import_module('common.handle.impl.text.' + module), converter)()
images = {}
def capture(files):
for file in files or []:
raw = file.meta.get('content')
if isinstance(raw, bytes):
mime = mimetypes.guess_type(file.file_name)[0] or 'image/jpeg'
images['./oss/file/' + str(file.id)] = f'data:{mime};base64,' + base64.b64encode(raw).decode()
upload = SimpleUploadedFile(Path(name.replace('\\', '/')).name, content)
evidence = {}
if suffix == 'docx':
from docx import Document
from .native_outline import word_markdown
# Use the converter's Markdown before knowledge-base chunking flattens headings.
markdown, evidence = word_markdown(Document(io.BytesIO(content)), handler, capture)
elif suffix in ('html', 'htm'):
from html2text import html2text
from common.handle.impl.text.html_split_handle import get_encoding
markdown = html2text(content.decode(get_encoding(content)))
from .native_outline import html_evidence
evidence = html_evidence(content)
elif suffix == 'pdf':
from .native_outline import pdf_markdown
markdown, evidence = pdf_markdown(content)
if markdown is None:
result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0)
markdown = '\n\n'.join(item.get('content', '') for item in result.get('content', []))
else:
result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0)
sections = result if isinstance(result, list) else [result]
markdown = '\n\n'.join('\n\n'.join(
((('# ' + item.get('title', '') + '\n') if item.get('title') else '') + item.get('content', ''))
for item in section.get('content', [])) for section in sections if isinstance(section, dict))
for path, uri in images.items():
markdown = markdown.replace(path, uri)
return {'markdown': text(markdown, '解析结果'), 'images': {},
'source': {'engine': 'maxkb-native', 'format': suffix, **evidence}}