259 lines
12 KiB
Python
259 lines
12 KiB
Python
"""Model file assets, independent of runtime/model connection records. Never executes model code."""
|
||
import json
|
||
import hashlib
|
||
import os
|
||
import shutil
|
||
import stat
|
||
import tempfile
|
||
import uuid
|
||
import zipfile
|
||
from contextlib import contextmanager
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path, PurePosixPath
|
||
|
||
MAX_BYTES = 2 * 1024 ** 3
|
||
MAX_FILES = 10000
|
||
WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'}
|
||
|
||
|
||
class ImportError(ValueError):
|
||
pass
|
||
|
||
|
||
def model_fingerprint(directory):
|
||
"""Hash relative paths and complete file contents, independent of the archive name."""
|
||
root = Path(directory)
|
||
digest = hashlib.sha256()
|
||
for file in sorted((p for p in root.rglob('*') if p.is_file()), key=lambda p: p.relative_to(root).as_posix()):
|
||
name = file.relative_to(root).as_posix().encode('utf-8')
|
||
digest.update(len(name).to_bytes(8, 'big'))
|
||
digest.update(name)
|
||
digest.update(file.stat().st_size.to_bytes(8, 'big'))
|
||
with file.open('rb') as source:
|
||
for chunk in iter(lambda: source.read(1024 * 1024), b''):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|
||
|
||
@contextmanager
|
||
def import_lock(root):
|
||
"""Serialize duplicate checks and record creation across Web processes."""
|
||
with (root / '.import.lock').open('a+b') as lock:
|
||
if os.name == 'nt':
|
||
import msvcrt
|
||
lock.seek(0)
|
||
msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
|
||
else:
|
||
import fcntl
|
||
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
||
try:
|
||
yield
|
||
finally:
|
||
if os.name == 'nt':
|
||
lock.seek(0)
|
||
msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
|
||
else:
|
||
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
|
||
|
||
|
||
def inspect_model(directory):
|
||
root = Path(directory)
|
||
if not root.is_dir() or root.is_symlink():
|
||
raise ImportError('请选择有效模型目录')
|
||
files = []
|
||
for current, dirs, names in os.walk(root, followlinks=False):
|
||
for name in dirs + names:
|
||
if (Path(current) / name).is_symlink():
|
||
raise ImportError('模型目录不能包含符号链接')
|
||
for name in names:
|
||
file = Path(current) / name
|
||
if not file.is_file():
|
||
raise ImportError('模型目录包含非普通文件')
|
||
files.append(file)
|
||
if len(files) > MAX_FILES:
|
||
raise ImportError('模型文件数量超过限制')
|
||
weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0]
|
||
if not weights:
|
||
raise ImportError('未找到有效模型权重文件')
|
||
|
||
def read_json(name):
|
||
file = root / name
|
||
if not file.exists():
|
||
return {}
|
||
if file.stat().st_size > 2 * 1024 ** 2:
|
||
raise ImportError(name + '超过大小限制')
|
||
try:
|
||
data = json.loads(file.read_text(encoding='utf-8'))
|
||
if not isinstance(data, dict):
|
||
raise ValueError()
|
||
return data
|
||
except (ValueError, UnicodeError):
|
||
raise ImportError(name + '格式错误')
|
||
|
||
config = read_json('config.json')
|
||
manifest = read_json('model-manifest.json')
|
||
quant = config.get('quantization_config') or {}
|
||
if not isinstance(quant, dict):
|
||
raise ImportError('量化配置格式错误')
|
||
quantization = str(quant.get('quant_method', '')).lower()
|
||
suffixes = {f.suffix.lower() for f in weights}
|
||
if config and suffixes & {'.safetensors', '.bin'}:
|
||
if not config.get('model_type') and not config.get('architectures'):
|
||
raise ImportError('config.json缺少模型架构信息')
|
||
format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF')
|
||
elif '.onnx' in suffixes:
|
||
format_name = 'ONNX'
|
||
elif '.gguf' in suffixes:
|
||
format_name = 'GGUF'
|
||
elif manifest:
|
||
format_name = 'CUSTOM'
|
||
else:
|
||
raise ImportError('模型目录缺少config.json或model-manifest.json')
|
||
# Validate referenced shards without trusting paths supplied by model files.
|
||
for index in root.glob('*.index.json'):
|
||
mapping = read_json(index.name).get('weight_map')
|
||
if not isinstance(mapping, dict) or not mapping:
|
||
raise ImportError('权重索引格式错误')
|
||
for name in mapping.values():
|
||
if not isinstance(name, str) or '\\' in name:
|
||
raise ImportError('权重分片路径无效')
|
||
relative = PurePosixPath(name)
|
||
if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file():
|
||
raise ImportError('权重分片缺失或路径无效')
|
||
return {
|
||
'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files),
|
||
'file_count': len(files), 'weight_count': len(weights),
|
||
'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config},
|
||
'quantization': quantization,
|
||
'checks': ['权重文件存在', '配置JSON及分片引用检查通过'],
|
||
'runtime_status': 'NOT_CHECKED',
|
||
'runtime_note': '文件已导入,运行能力需接入验证',
|
||
}
|
||
|
||
|
||
class ModelImportStore:
|
||
def __init__(self, root):
|
||
self.root = Path(root)
|
||
self.root.mkdir(parents=True, exist_ok=True)
|
||
|
||
def _records(self):
|
||
records = []
|
||
for path in self.root.glob('*/import.json'):
|
||
if path.parent.name.startswith('.import-'):
|
||
continue
|
||
try:
|
||
records.append(json.loads(path.read_text(encoding='utf-8')))
|
||
except (ValueError, OSError):
|
||
continue
|
||
return sorted(records, key=lambda r: r.get('created_at', ''), reverse=True)
|
||
|
||
def list(self):
|
||
records, paths, fingerprints = [], set(), set()
|
||
for record in self._records():
|
||
path = record.get('path')
|
||
canonical = os.path.normcase(os.path.realpath(path)) if path else None
|
||
fingerprint = record.get('fingerprint') if record.get('source') == 'UPLOAD' else None
|
||
if (canonical and canonical in paths) or (fingerprint and fingerprint in fingerprints):
|
||
continue
|
||
if canonical:
|
||
paths.add(canonical)
|
||
if fingerprint:
|
||
fingerprints.add(fingerprint)
|
||
records.append(record)
|
||
if len(records) == 200:
|
||
break
|
||
return records
|
||
|
||
def create(self, owner_id, path=None, upload=None):
|
||
if (path is None) == (upload is None):
|
||
raise ImportError('请选择一种导入方式')
|
||
identity = str(uuid.uuid4())
|
||
staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root))
|
||
try:
|
||
if upload is not None:
|
||
if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES:
|
||
raise ImportError('请上传2GB以内的ZIP模型包')
|
||
archive = staging / 'upload.zip'
|
||
with archive.open('wb') as output:
|
||
size = 0
|
||
for chunk in upload.chunks():
|
||
size += len(chunk)
|
||
if size > MAX_BYTES:
|
||
raise ImportError('模型包超过2GB')
|
||
output.write(chunk)
|
||
source = staging / 'files'
|
||
source.mkdir()
|
||
try:
|
||
with zipfile.ZipFile(archive) as zipped:
|
||
members = zipped.infolist()
|
||
if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES:
|
||
raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录')
|
||
seen = set()
|
||
for member in members:
|
||
name = PurePosixPath(member.filename)
|
||
mode = member.external_attr >> 16
|
||
if (name.is_absolute() or '..' in name.parts or '\\' in member.filename
|
||
or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1
|
||
or member.filename in seen):
|
||
raise ImportError('模型包包含不安全的路径或文件')
|
||
seen.add(member.filename)
|
||
target = source.joinpath(*name.parts)
|
||
if member.is_dir():
|
||
target.mkdir(parents=True, exist_ok=True)
|
||
else:
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipped.open(member) as incoming, target.open('xb') as output:
|
||
shutil.copyfileobj(incoming, output, 1024 * 1024)
|
||
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
|
||
raise ImportError('模型包损坏或无法解压') from exc
|
||
archive.unlink()
|
||
# Most model archives contain one enclosing directory.
|
||
children = list(source.iterdir())
|
||
model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source
|
||
info = inspect_model(model_dir)
|
||
final_path = self.root / identity / model_dir.relative_to(staging)
|
||
source_type = 'UPLOAD'
|
||
display_name = Path(upload.name).stem
|
||
else:
|
||
if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute():
|
||
raise ImportError('请输入服务器绝对目录')
|
||
model_dir = Path(path).resolve(strict=True)
|
||
info = inspect_model(model_dir)
|
||
final_path, source_type, display_name = model_dir, 'PATH', model_dir.name
|
||
with import_lock(self.root):
|
||
existing = self._records()
|
||
if source_type == 'PATH':
|
||
canonical = os.path.normcase(os.path.realpath(model_dir))
|
||
for record in existing:
|
||
if record.get('path') and os.path.normcase(os.path.realpath(record['path'])) == canonical:
|
||
return {**record, 'already_imported': True}
|
||
fingerprint = model_fingerprint(model_dir)
|
||
for record in existing:
|
||
if (record.get('size_bytes') != info['size_bytes'] or
|
||
record.get('file_count') != info['file_count']):
|
||
continue
|
||
previous_path = record.get('path')
|
||
if not previous_path or not Path(previous_path).is_dir():
|
||
continue
|
||
try:
|
||
inspect_model(previous_path)
|
||
previous = model_fingerprint(previous_path)
|
||
except (OSError, ImportError):
|
||
continue
|
||
if previous == fingerprint:
|
||
return {**record, 'already_imported': True}
|
||
record = {
|
||
'id': identity, 'name': display_name, 'path': str(final_path),
|
||
'source': source_type, 'owner_id': str(owner_id),
|
||
'created_at': datetime.now(timezone.utc).isoformat(),
|
||
'status': 'IMPORTED', 'fingerprint': fingerprint, **info,
|
||
}
|
||
(staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8')
|
||
staging.rename(self.root / identity)
|
||
return record
|
||
except (FileNotFoundError, PermissionError) as exc:
|
||
raise ImportError('目录不存在或无访问权限') from exc
|
||
finally:
|
||
if staging.exists():
|
||
shutil.rmtree(staging)
|