166 lines
7.7 KiB
Python
166 lines
7.7 KiB
Python
"""MinerU 4 V1 upload/job/file protocol, independent of convenience routes."""
|
||
import base64
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import mimetypes
|
||
import re
|
||
import time
|
||
import zipfile
|
||
from pathlib import PurePosixPath
|
||
from urllib.parse import urljoin, urlsplit
|
||
import requests
|
||
from rest_framework.exceptions import ValidationError
|
||
|
||
MAX_OUTPUT = 150 * 1024 * 1024
|
||
|
||
|
||
def origin(url):
|
||
p = urlsplit(url)
|
||
if p.scheme not in ('http', 'https') or not p.hostname or p.username or p.password:
|
||
raise ValidationError('解析服务返回地址无效')
|
||
return p.scheme.lower(), p.hostname.lower(), p.port or (443 if p.scheme == 'https' else 80)
|
||
|
||
|
||
def identifier(value):
|
||
if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,200}', value):
|
||
raise ValidationError('解析服务返回资源 ID 无效')
|
||
return value
|
||
|
||
|
||
def read(response):
|
||
chunks, size = [], 0
|
||
try:
|
||
for chunk in response.iter_content(65536):
|
||
size += len(chunk)
|
||
if size > MAX_OUTPUT:
|
||
raise ValidationError('解析结果超过大小限制')
|
||
chunks.append(chunk)
|
||
finally:
|
||
response.close()
|
||
return b''.join(chunks)
|
||
|
||
|
||
def call(method, url, headers, expect_json=True, **kwargs):
|
||
response = requests.request(method, url, headers=headers, timeout=(10, 30), allow_redirects=False,
|
||
stream=True, **kwargs)
|
||
if response.status_code not in (200, 201, 202, 204):
|
||
response.close()
|
||
raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})')
|
||
raw = read(response)
|
||
if not expect_json or response.status_code == 204 or not raw:
|
||
return {}
|
||
try:
|
||
result = json.loads(raw)
|
||
except (ValueError, UnicodeDecodeError) as exc:
|
||
raise ValidationError('解析服务返回格式无效') from exc
|
||
if not isinstance(result, dict):
|
||
raise ValidationError('解析服务返回格式无效')
|
||
return result
|
||
|
||
|
||
def download(url, base, headers):
|
||
for _ in range(4):
|
||
response = requests.get(url, headers=headers if origin(url) == origin(base) else {},
|
||
timeout=(10, 30), allow_redirects=False, stream=True)
|
||
if response.status_code in (301, 302, 303, 307, 308):
|
||
target = response.headers.get('Location', '')
|
||
response.close()
|
||
if not target:
|
||
raise ValidationError('解析结果跳转地址无效')
|
||
url = urljoin(url, target)
|
||
origin(url)
|
||
continue
|
||
if response.status_code != 200:
|
||
response.close()
|
||
raise ValidationError(f'获取解析结果失败(HTTP {response.status_code})')
|
||
return read(response)
|
||
raise ValidationError('解析结果跳转次数过多')
|
||
|
||
|
||
def unpack(raw):
|
||
images = {}
|
||
try:
|
||
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
|
||
if sum(i.file_size for i in archive.infolist()) > 200 * 1024 * 1024:
|
||
raise ValidationError('解析结果解压后超过大小限制')
|
||
for item in archive.infolist():
|
||
path = PurePosixPath(item.filename)
|
||
if '..' in path.parts or path.is_absolute():
|
||
raise ValidationError('解析结果路径无效')
|
||
mime = mimetypes.guess_type(item.filename)[0] or ''
|
||
if mime.startswith('image/'):
|
||
images[item.filename] = 'data:' + mime + ';base64,' + base64.b64encode(archive.read(item)).decode()
|
||
except (zipfile.BadZipFile, RuntimeError) as exc:
|
||
raise ValidationError('解析结果压缩包无效') from exc
|
||
return images
|
||
|
||
|
||
def parse(parser, name, content, options, headers, progress_callback=None):
|
||
from system_manage.ai_hub import text, MAX_INLINE_TEXT
|
||
from system_manage.ai_hub_progress import parser_progress
|
||
endpoint = parser.endpoint.rstrip('/')
|
||
base = endpoint if endpoint.endswith('/v1') else endpoint + '/v1'
|
||
root = base[:-3]
|
||
deadline = time.monotonic() + 300
|
||
try:
|
||
upload = call('POST', base + '/uploads', headers, json={
|
||
'filename': name, 'bytes': len(content), 'mime_type': mimetypes.guess_type(name)[0] or 'application/octet-stream',
|
||
'purpose': 'parse', 'sha256sum': hashlib.sha256(content).hexdigest()})
|
||
if upload.get('status') == 'pending':
|
||
uid = identifier(upload.get('id'))
|
||
target = urljoin(root + '/', upload.get('upload_url', ''))
|
||
if not upload.get('upload_url') or upload.get('upload_method', 'PUT') != 'PUT':
|
||
raise ValidationError('解析上传协议无效')
|
||
origin(target)
|
||
extra = upload.get('upload_headers') or {}
|
||
if not isinstance(extra, dict) or any(not isinstance(k, str) or not isinstance(v, str) or '\n' in k+v or '\r' in k+v for k,v in extra.items()):
|
||
raise ValidationError('解析上传请求头无效')
|
||
# Never forward the registered parser credential to object storage.
|
||
auth = {**extra, **headers} if origin(target) == origin(base) else extra
|
||
call('PUT', target, auth, expect_json=False, data=content)
|
||
upload = call('POST', base + '/uploads/' + uid + '/complete', headers)
|
||
if upload.get('status') != 'completed':
|
||
raise ValidationError('解析文件上传未完成')
|
||
fid = identifier((upload.get('file') or {}).get('id'))
|
||
job = call('POST', base + '/parse/jobs', headers, json={
|
||
'files': [{'source': {'type': 'file_id', 'file_id': fid}}],
|
||
'tier': options.get('tier', 'standard'), 'ocr_mode': 'ocr' if options.get('ocr') else 'auto',
|
||
'output_formats': ['markdown', 'middle_json', 'zip']})
|
||
jid = identifier(job.get('job_id'))
|
||
while True:
|
||
state = job.get('status')
|
||
if state not in ('queued', 'running', 'completed', 'partial', 'failed', 'canceled'):
|
||
raise ValidationError('解析任务状态无效')
|
||
if progress_callback:
|
||
progress_callback(parser_progress(job))
|
||
if state not in ('queued', 'running'):
|
||
break
|
||
if time.monotonic() >= deadline:
|
||
raise ValidationError('解析等待超时,后台任务 ID:' + jid)
|
||
time.sleep(2)
|
||
job = call('GET', base + '/parse/jobs/' + jid, headers)
|
||
files = job.get('files')
|
||
if state != 'completed' or not isinstance(files, list) or len(files) != 1 or files[0].get('status') != 'completed':
|
||
raise ValidationError('解析任务未成功,后台任务 ID:' + jid)
|
||
outputs = files[0].get('output_files') or {}
|
||
def artifact(kind):
|
||
ref = outputs.get(kind)
|
||
if not isinstance(ref, dict):
|
||
raise ValidationError('解析结果缺少 ' + kind)
|
||
return download(base + '/files/' + identifier(ref.get('file_id')) + '/content', base, headers)
|
||
markdown = artifact('markdown').decode('utf-8')
|
||
middle = json.loads(artifact('middle_json'))
|
||
images = unpack(artifact('zip')) if 'zip' in outputs else {}
|
||
# ZIP outputs can wrap a document directory; include relative image paths.
|
||
for path, value in list(images.items()):
|
||
parts = PurePosixPath(path).parts
|
||
if 'images' in parts:
|
||
images['/'.join(parts[parts.index('images'):])] = value
|
||
return {'markdown': text(markdown, '解析结果', limit=MAX_INLINE_TEXT), 'parser_id': str(parser.id),
|
||
'images': images, 'source': {'task_id': jid, 'middle_json': middle}}
|
||
except requests.RequestException as exc:
|
||
raise ValidationError('解析服务连接失败或超时') from exc
|
||
except (ValueError, UnicodeDecodeError) as exc:
|
||
raise ValidationError('解析结果内容无效') from exc
|