103 lines
5.3 KiB
Python
103 lines
5.3 KiB
Python
"""Reuse MaxKB's built-in converters; MinerU receives only supported visual documents."""
|
||
import base64
|
||
import io
|
||
import mimetypes
|
||
from pathlib import Path
|
||
import shutil
|
||
import subprocess
|
||
import tempfile
|
||
import zipfile
|
||
from rest_framework.exceptions import ValidationError
|
||
|
||
|
||
def parse(name, content):
|
||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||
from system_manage.ai_hub import text
|
||
from . import extension
|
||
suffix = extension(name)
|
||
if suffix in ('md', 'txt'):
|
||
# Preserve headings and whitespace even when a document has no body paragraph.
|
||
try:
|
||
markdown = content.decode('utf-8-sig')
|
||
except UnicodeDecodeError:
|
||
from charset_normalizer import from_bytes
|
||
decoded = from_bytes(content).best()
|
||
if decoded is None:
|
||
raise ValidationError('无法识别文本编码,请使用 UTF-8')
|
||
markdown = str(decoded)
|
||
return {'markdown': text(markdown, '解析结果'), 'images': {},
|
||
'source': {'engine': 'maxkb-native', 'format': suffix}}
|
||
if suffix == 'doc':
|
||
command = shutil.which('libreoffice') or shutil.which('soffice')
|
||
if command is None:
|
||
raise ValidationError('DOC 格式需要 LibreOffice,请先另存为 DOCX')
|
||
with tempfile.TemporaryDirectory(prefix='maxkb-office-') as directory:
|
||
root = Path(directory); source = root / 'input.doc'; source.write_bytes(content)
|
||
try:
|
||
result = subprocess.run([command, '-env:UserInstallation=' + (root / 'profile').as_uri(),
|
||
'--headless', '--convert-to', 'docx', '--outdir', directory, str(source)],
|
||
capture_output=True, timeout=120)
|
||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||
raise ValidationError('DOC 转换失败,请另存为 DOCX') from exc
|
||
output = root / 'input.docx'
|
||
if result.returncode or not output.is_file():
|
||
raise ValidationError('DOC 转换失败,请另存为 DOCX')
|
||
parsed = parse(Path(name).stem + '.docx', output.read_bytes())
|
||
parsed['source']['converted_from'] = 'doc'
|
||
return parsed
|
||
if suffix in ('docx', 'xlsx'):
|
||
try:
|
||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||
if sum(item.file_size for item in archive.infolist()) > 200 * 1024 * 1024:
|
||
raise ValidationError('文档解压后超过大小限制')
|
||
except zipfile.BadZipFile as exc:
|
||
raise ValidationError('文件格式无效') from exc
|
||
converters = {
|
||
'pdf': ('pdf_split_handle', 'PdfSplitHandle'),
|
||
'docx': ('doc_split_handle', 'DocSplitHandle'), 'xlsx': ('xlsx_split_handle', 'XlsxSplitHandle'),
|
||
'xls': ('xls_split_handle', 'XlsSplitHandle'), 'csv': ('csv_split_handle', 'CsvSplitHandle'),
|
||
'html': ('html_split_handle', 'HTMLSplitHandle'), 'htm': ('html_split_handle', 'HTMLSplitHandle'),
|
||
'md': ('text_split_handle', 'TextSplitHandle'), 'txt': ('text_split_handle', 'TextSplitHandle'),
|
||
}
|
||
if suffix not in converters:
|
||
raise ValidationError('内置处理器不支持该文件格式')
|
||
import importlib
|
||
module, converter = converters[suffix]
|
||
handler = getattr(importlib.import_module('common.handle.impl.text.' + module), converter)()
|
||
images = {}
|
||
def capture(files):
|
||
for file in files or []:
|
||
raw = file.meta.get('content')
|
||
if isinstance(raw, bytes):
|
||
mime = mimetypes.guess_type(file.file_name)[0] or 'image/jpeg'
|
||
images['./oss/file/' + str(file.id)] = f'data:{mime};base64,' + base64.b64encode(raw).decode()
|
||
upload = SimpleUploadedFile(Path(name.replace('\\', '/')).name, content)
|
||
evidence = {}
|
||
if suffix == 'docx':
|
||
from docx import Document
|
||
from .native_outline import word_markdown
|
||
# Use the converter's Markdown before knowledge-base chunking flattens headings.
|
||
markdown, evidence = word_markdown(Document(io.BytesIO(content)), handler, capture)
|
||
elif suffix in ('html', 'htm'):
|
||
from html2text import html2text
|
||
from common.handle.impl.text.html_split_handle import get_encoding
|
||
markdown = html2text(content.decode(get_encoding(content)))
|
||
from .native_outline import html_evidence
|
||
evidence = html_evidence(content)
|
||
elif suffix == 'pdf':
|
||
from .native_outline import pdf_markdown
|
||
markdown, evidence = pdf_markdown(content)
|
||
if markdown is None:
|
||
result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0)
|
||
markdown = '\n\n'.join(item.get('content', '') for item in result.get('content', []))
|
||
else:
|
||
result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0)
|
||
sections = result if isinstance(result, list) else [result]
|
||
markdown = '\n\n'.join('\n\n'.join(
|
||
((('# ' + item.get('title', '') + '\n') if item.get('title') else '') + item.get('content', ''))
|
||
for item in section.get('content', [])) for section in sections if isinstance(section, dict))
|
||
for path, uri in images.items():
|
||
markdown = markdown.replace(path, uri)
|
||
return {'markdown': text(markdown, '解析结果'), 'images': {},
|
||
'source': {'engine': 'maxkb-native', 'format': suffix, **evidence}}
|