UnisKB/apps/system_manage/ai_hub_parser_v1.py

166 lines
7.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""MinerU 4 V1 upload/job/file protocol, independent of convenience routes."""
import base64
import hashlib
import io
import json
import mimetypes
import re
import time
import zipfile
from pathlib import PurePosixPath
from urllib.parse import urljoin, urlsplit
import requests
from rest_framework.exceptions import ValidationError
MAX_OUTPUT = 150 * 1024 * 1024
def origin(url):
p = urlsplit(url)
if p.scheme not in ('http', 'https') or not p.hostname or p.username or p.password:
raise ValidationError('解析服务返回地址无效')
return p.scheme.lower(), p.hostname.lower(), p.port or (443 if p.scheme == 'https' else 80)
def identifier(value):
if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,200}', value):
raise ValidationError('解析服务返回资源 ID 无效')
return value
def read(response):
chunks, size = [], 0
try:
for chunk in response.iter_content(65536):
size += len(chunk)
if size > MAX_OUTPUT:
raise ValidationError('解析结果超过大小限制')
chunks.append(chunk)
finally:
response.close()
return b''.join(chunks)
def call(method, url, headers, expect_json=True, **kwargs):
response = requests.request(method, url, headers=headers, timeout=(10, 30), allow_redirects=False,
stream=True, **kwargs)
if response.status_code not in (200, 201, 202, 204):
response.close()
raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})')
raw = read(response)
if not expect_json or response.status_code == 204 or not raw:
return {}
try:
result = json.loads(raw)
except (ValueError, UnicodeDecodeError) as exc:
raise ValidationError('解析服务返回格式无效') from exc
if not isinstance(result, dict):
raise ValidationError('解析服务返回格式无效')
return result
def download(url, base, headers):
for _ in range(4):
response = requests.get(url, headers=headers if origin(url) == origin(base) else {},
timeout=(10, 30), allow_redirects=False, stream=True)
if response.status_code in (301, 302, 303, 307, 308):
target = response.headers.get('Location', '')
response.close()
if not target:
raise ValidationError('解析结果跳转地址无效')
url = urljoin(url, target)
origin(url)
continue
if response.status_code != 200:
response.close()
raise ValidationError(f'获取解析结果失败(HTTP {response.status_code})')
return read(response)
raise ValidationError('解析结果跳转次数过多')
def unpack(raw):
images = {}
try:
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
if sum(i.file_size for i in archive.infolist()) > 200 * 1024 * 1024:
raise ValidationError('解析结果解压后超过大小限制')
for item in archive.infolist():
path = PurePosixPath(item.filename)
if '..' in path.parts or path.is_absolute():
raise ValidationError('解析结果路径无效')
mime = mimetypes.guess_type(item.filename)[0] or ''
if mime.startswith('image/'):
images[item.filename] = 'data:' + mime + ';base64,' + base64.b64encode(archive.read(item)).decode()
except (zipfile.BadZipFile, RuntimeError) as exc:
raise ValidationError('解析结果压缩包无效') from exc
return images
def parse(parser, name, content, options, headers, progress_callback=None):
from system_manage.ai_hub import text, MAX_INLINE_TEXT
from system_manage.ai_hub_progress import parser_progress
endpoint = parser.endpoint.rstrip('/')
base = endpoint if endpoint.endswith('/v1') else endpoint + '/v1'
root = base[:-3]
deadline = time.monotonic() + 300
try:
upload = call('POST', base + '/uploads', headers, json={
'filename': name, 'bytes': len(content), 'mime_type': mimetypes.guess_type(name)[0] or 'application/octet-stream',
'purpose': 'parse', 'sha256sum': hashlib.sha256(content).hexdigest()})
if upload.get('status') == 'pending':
uid = identifier(upload.get('id'))
target = urljoin(root + '/', upload.get('upload_url', ''))
if not upload.get('upload_url') or upload.get('upload_method', 'PUT') != 'PUT':
raise ValidationError('解析上传协议无效')
origin(target)
extra = upload.get('upload_headers') or {}
if not isinstance(extra, dict) or any(not isinstance(k, str) or not isinstance(v, str) or '\n' in k+v or '\r' in k+v for k,v in extra.items()):
raise ValidationError('解析上传请求头无效')
# Never forward the registered parser credential to object storage.
auth = {**extra, **headers} if origin(target) == origin(base) else extra
call('PUT', target, auth, expect_json=False, data=content)
upload = call('POST', base + '/uploads/' + uid + '/complete', headers)
if upload.get('status') != 'completed':
raise ValidationError('解析文件上传未完成')
fid = identifier((upload.get('file') or {}).get('id'))
job = call('POST', base + '/parse/jobs', headers, json={
'files': [{'source': {'type': 'file_id', 'file_id': fid}}],
'tier': options.get('tier', 'standard'), 'ocr_mode': 'ocr' if options.get('ocr') else 'auto',
'output_formats': ['markdown', 'middle_json', 'zip']})
jid = identifier(job.get('job_id'))
while True:
state = job.get('status')
if state not in ('queued', 'running', 'completed', 'partial', 'failed', 'canceled'):
raise ValidationError('解析任务状态无效')
if progress_callback:
progress_callback(parser_progress(job))
if state not in ('queued', 'running'):
break
if time.monotonic() >= deadline:
raise ValidationError('解析等待超时,后台任务 ID:' + jid)
time.sleep(2)
job = call('GET', base + '/parse/jobs/' + jid, headers)
files = job.get('files')
if state != 'completed' or not isinstance(files, list) or len(files) != 1 or files[0].get('status') != 'completed':
raise ValidationError('解析任务未成功,后台任务 ID:' + jid)
outputs = files[0].get('output_files') or {}
def artifact(kind):
ref = outputs.get(kind)
if not isinstance(ref, dict):
raise ValidationError('解析结果缺少 ' + kind)
return download(base + '/files/' + identifier(ref.get('file_id')) + '/content', base, headers)
markdown = artifact('markdown').decode('utf-8')
middle = json.loads(artifact('middle_json'))
images = unpack(artifact('zip')) if 'zip' in outputs else {}
# ZIP outputs can wrap a document directory; include relative image paths.
for path, value in list(images.items()):
parts = PurePosixPath(path).parts
if 'images' in parts:
images['/'.join(parts[parts.index('images'):])] = value
return {'markdown': text(markdown, '解析结果', limit=MAX_INLINE_TEXT), 'parser_id': str(parser.id),
'images': images, 'source': {'task_id': jid, 'middle_json': middle}}
except requests.RequestException as exc:
raise ValidationError('解析服务连接失败或超时') from exc
except (ValueError, UnicodeDecodeError) as exc:
raise ValidationError('解析结果内容无效') from exc