214 lines
8.4 KiB
Python
214 lines
8.4 KiB
Python
"""Private, durable image files. References live in existing task/artifact records.
|
|
|
|
AI_HUB_ASSET_DIR must be on persistent shared storage when Web/worker are on
|
|
different hosts. No migration, public static route, or Redis image payloads.
|
|
"""
|
|
import base64
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from urllib.parse import quote
|
|
|
|
from django.conf import settings
|
|
from rest_framework.exceptions import NotFound, ValidationError
|
|
|
|
DATA_IMAGE = re.compile(r'data:image/(?:png|jpe?g|gif|webp|bmp);base64,[A-Za-z0-9+/=\r\n]+', re.I)
|
|
ASSET = re.compile(r'/admin/api/workspace/([^/\s"<>]+)/ai_hub/assets/([a-f0-9-]{36})/content')
|
|
MAX_IMAGE = 20 * 1024 * 1024
|
|
MAX_IMAGES_BYTES = 100 * 1024 * 1024
|
|
TTL = 7 * 86400
|
|
|
|
|
|
def root():
|
|
return Path(os.environ.get('AI_HUB_ASSET_DIR') or getattr(settings, 'AI_HUB_ASSET_DIR', '')
|
|
or Path(__file__).resolve().parents[2] / 'data' / 'ai-hub-assets').resolve()
|
|
|
|
|
|
@contextmanager
|
|
def asset_lock():
|
|
directory = root()
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
lock = directory / '.lock'
|
|
stream = lock.open('a+b')
|
|
stream.seek(0); stream.write(b'0'); stream.flush(); stream.seek(0)
|
|
deadline = time.monotonic() + 30
|
|
while True:
|
|
try:
|
|
if os.name == 'nt':
|
|
import msvcrt
|
|
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
break
|
|
except OSError:
|
|
if time.monotonic() >= deadline:
|
|
stream.close()
|
|
raise ValidationError('图片存储繁忙,请稍后重试')
|
|
time.sleep(.05)
|
|
try:
|
|
yield
|
|
finally:
|
|
if os.name == 'nt':
|
|
stream.seek(0)
|
|
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
|
else:
|
|
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
|
stream.close()
|
|
|
|
|
|
def _write(path, content):
|
|
temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
|
|
try:
|
|
with temporary.open('wb') as stream:
|
|
stream.write(content)
|
|
os.replace(temporary, path)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _metadata(asset_id):
|
|
try:
|
|
asset_id = str(uuid.UUID(str(asset_id)))
|
|
return json.loads((root() / (asset_id + '.json')).read_text(encoding='utf-8'))
|
|
except (OSError, ValueError, TypeError):
|
|
raise NotFound('图片不存在或已过期')
|
|
|
|
|
|
def authorize(workspace, user, asset_id):
|
|
from system_manage.ai_hub import admin
|
|
meta = _metadata(asset_id)
|
|
if meta['workspace_id'] != workspace or (meta['owner_id'] != str(user.id) and not admin(user)):
|
|
raise NotFound('图片不存在或无权访问')
|
|
return meta
|
|
|
|
|
|
def image_bytes(workspace, user, asset_id):
|
|
meta = authorize(workspace, user, asset_id)
|
|
try:
|
|
content = (root() / (str(uuid.UUID(str(asset_id))) + '.bin')).read_bytes()
|
|
except OSError:
|
|
raise NotFound('图片不存在或已过期')
|
|
return content, meta
|
|
|
|
|
|
def _store(uri, workspace, user):
|
|
from PIL import Image, UnidentifiedImageError
|
|
try:
|
|
encoded = uri.split(',', 1)[1]
|
|
if len(encoded) > MAX_IMAGE * 4 // 3 + 16:
|
|
raise ValueError()
|
|
content = base64.b64decode(re.sub(r'\s', '', encoded), validate=True)
|
|
if not content or len(content) > MAX_IMAGE:
|
|
raise ValueError()
|
|
with Image.open(io.BytesIO(content)) as image:
|
|
if image.width * image.height > 40_000_000:
|
|
raise ValueError()
|
|
fmt = image.format
|
|
image.verify()
|
|
mime = {'PNG': 'image/png', 'JPEG': 'image/jpeg', 'GIF': 'image/gif',
|
|
'WEBP': 'image/webp', 'BMP': 'image/bmp'}[fmt]
|
|
except (ValueError, KeyError, OSError, UnidentifiedImageError, Image.DecompressionBombError):
|
|
raise ValidationError('图片格式无效或超过大小限制')
|
|
digest = hashlib.sha256(content).hexdigest()
|
|
asset_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f'ai-hub:{workspace}:{user.id}:{digest}'))
|
|
meta = {'id': asset_id, 'workspace_id': workspace, 'owner_id': str(user.id),
|
|
'sha256': digest, 'mime': mime, 'size': len(content), 'touched_at': time.time()}
|
|
if not (root() / (asset_id + '.bin')).exists():
|
|
_write(root() / (asset_id + '.bin'), content)
|
|
_write(root() / (asset_id + '.json'), json.dumps(meta).encode())
|
|
return f'/admin/api/workspace/{quote(workspace, safe="")}/ai_hub/assets/{asset_id}/content'
|
|
|
|
|
|
def normalize(markdown, workspace, user, images=None):
|
|
"""Caller holds asset_lock until its database reference is saved."""
|
|
replacements = {}
|
|
total = 0
|
|
|
|
def store(uri):
|
|
nonlocal total
|
|
if uri not in replacements:
|
|
total += len(uri) * 3 // 4
|
|
if total > MAX_IMAGES_BYTES:
|
|
raise ValidationError('文档图片总大小超过限制')
|
|
replacements[uri] = _store(uri, workspace, user)
|
|
return replacements[uri]
|
|
|
|
if images:
|
|
if not isinstance(images, dict):
|
|
raise ValidationError('解析图片格式无效')
|
|
# Only replace destinations in Markdown/HTML, never arbitrary words.
|
|
for name, uri in images.items():
|
|
if not isinstance(name, str) or not isinstance(uri, str) or not DATA_IMAGE.fullmatch(uri):
|
|
continue
|
|
pattern = re.compile(r'(?P<prefix>\]\(<?|src=["\x27]|^\s*\[[^\]\n]+\]:\s*<?)(?:\./)?(?:images/)?'
|
|
+ re.escape(name.removeprefix('images/')) + r'(?=[\s)>"\x27])', re.M)
|
|
if pattern.search(markdown):
|
|
url = store(uri)
|
|
markdown = pattern.sub(lambda m: m['prefix'] + url, markdown)
|
|
markdown = DATA_IMAGE.sub(lambda m: store(m[0]), markdown)
|
|
for match in ASSET.finditer(markdown):
|
|
if match[1] != quote(workspace, safe=''):
|
|
raise ValidationError('不能引用其他工作空间的图片')
|
|
meta = authorize(workspace, user, match[2])
|
|
meta['touched_at'] = time.time()
|
|
_write(root() / (match[2] + '.json'), json.dumps(meta).encode())
|
|
return markdown
|
|
|
|
|
|
def protect_images(markdown):
|
|
"""Keep image destinations out of model rewriting; reject lost/forged tokens."""
|
|
images = []
|
|
pattern = re.compile(r'!\[[^\]\n]*\]\([^\n]*?\)|<img\b[^>]*>', re.I)
|
|
def replace(match):
|
|
images.append(match[0])
|
|
return f'AIHUBIMAGE{len(images):04d}TOKEN'
|
|
return pattern.sub(replace, markdown), images
|
|
|
|
|
|
def restore_images(markdown, images, append_missing=True):
|
|
# Summarizers may omit or repeat placeholders. The backend owns the image
|
|
# manifest, so preserve known positions and append omitted originals once.
|
|
known = {str(index + 1): image for index, image in enumerate(images)}
|
|
restored = set()
|
|
def replace(match):
|
|
index = match[1].lstrip('0') or '0'
|
|
if index not in known or index in restored:
|
|
return ''
|
|
restored.add(index)
|
|
return known[index]
|
|
markdown = re.sub(r'AIHUBIMAGE[\s_*`-]*(\d+)[\s_*`-]*TOKEN', replace, markdown, flags=re.I)
|
|
missing = [image for index, image in known.items() if index not in restored]
|
|
if missing and append_missing:
|
|
markdown = markdown.rstrip() + '\n\n## 原文图片\n\n' + '\n\n'.join(missing)
|
|
return markdown
|
|
|
|
|
|
def cleanup(apply=False, now=None):
|
|
"""Only expired, unreferenced Hub images; saved originals/history count as refs."""
|
|
from system_manage.models.ai_hub import AIArtifact, AITask
|
|
cutoff = (now or time.time()) - TTL
|
|
expired = []
|
|
with asset_lock():
|
|
protected = set()
|
|
for artifact in AIArtifact.objects.only('markdown', 'original', 'history').iterator():
|
|
content = artifact.markdown + '\n' + artifact.original + '\n' + json.dumps(artifact.history)
|
|
protected.update(m[2] for m in ASSET.finditer(content))
|
|
for task in AITask.objects.filter(status__in=['PENDING', 'RUNNING']).only('input_data', 'result').iterator():
|
|
protected.update(m[2] for m in ASSET.finditer(json.dumps([task.input_data, task.result])))
|
|
for path in root().glob('*.json'):
|
|
meta = _metadata(path.stem)
|
|
if meta['touched_at'] >= cutoff or path.stem in protected:
|
|
continue
|
|
expired.append(path.stem)
|
|
if apply:
|
|
(root() / (path.stem + '.bin')).unlink(missing_ok=True)
|
|
path.unlink()
|
|
return len(expired)
|