UnisKB/apps/models_provider/local_imports.py

259 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Model file assets, independent of runtime/model connection records. Never executes model code."""
import json
import hashlib
import os
import shutil
import stat
import tempfile
import uuid
import zipfile
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
MAX_BYTES = 2 * 1024 ** 3
MAX_FILES = 10000
WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'}
class ImportError(ValueError):
pass
def model_fingerprint(directory):
"""Hash relative paths and complete file contents, independent of the archive name."""
root = Path(directory)
digest = hashlib.sha256()
for file in sorted((p for p in root.rglob('*') if p.is_file()), key=lambda p: p.relative_to(root).as_posix()):
name = file.relative_to(root).as_posix().encode('utf-8')
digest.update(len(name).to_bytes(8, 'big'))
digest.update(name)
digest.update(file.stat().st_size.to_bytes(8, 'big'))
with file.open('rb') as source:
for chunk in iter(lambda: source.read(1024 * 1024), b''):
digest.update(chunk)
return digest.hexdigest()
@contextmanager
def import_lock(root):
"""Serialize duplicate checks and record creation across Web processes."""
with (root / '.import.lock').open('a+b') as lock:
if os.name == 'nt':
import msvcrt
lock.seek(0)
msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
else:
import fcntl
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
if os.name == 'nt':
lock.seek(0)
msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
else:
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
def inspect_model(directory):
root = Path(directory)
if not root.is_dir() or root.is_symlink():
raise ImportError('请选择有效模型目录')
files = []
for current, dirs, names in os.walk(root, followlinks=False):
for name in dirs + names:
if (Path(current) / name).is_symlink():
raise ImportError('模型目录不能包含符号链接')
for name in names:
file = Path(current) / name
if not file.is_file():
raise ImportError('模型目录包含非普通文件')
files.append(file)
if len(files) > MAX_FILES:
raise ImportError('模型文件数量超过限制')
weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0]
if not weights:
raise ImportError('未找到有效模型权重文件')
def read_json(name):
file = root / name
if not file.exists():
return {}
if file.stat().st_size > 2 * 1024 ** 2:
raise ImportError(name + '超过大小限制')
try:
data = json.loads(file.read_text(encoding='utf-8'))
if not isinstance(data, dict):
raise ValueError()
return data
except (ValueError, UnicodeError):
raise ImportError(name + '格式错误')
config = read_json('config.json')
manifest = read_json('model-manifest.json')
quant = config.get('quantization_config') or {}
if not isinstance(quant, dict):
raise ImportError('量化配置格式错误')
quantization = str(quant.get('quant_method', '')).lower()
suffixes = {f.suffix.lower() for f in weights}
if config and suffixes & {'.safetensors', '.bin'}:
if not config.get('model_type') and not config.get('architectures'):
raise ImportError('config.json缺少模型架构信息')
format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF')
elif '.onnx' in suffixes:
format_name = 'ONNX'
elif '.gguf' in suffixes:
format_name = 'GGUF'
elif manifest:
format_name = 'CUSTOM'
else:
raise ImportError('模型目录缺少config.json或model-manifest.json')
# Validate referenced shards without trusting paths supplied by model files.
for index in root.glob('*.index.json'):
mapping = read_json(index.name).get('weight_map')
if not isinstance(mapping, dict) or not mapping:
raise ImportError('权重索引格式错误')
for name in mapping.values():
if not isinstance(name, str) or '\\' in name:
raise ImportError('权重分片路径无效')
relative = PurePosixPath(name)
if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file():
raise ImportError('权重分片缺失或路径无效')
return {
'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files),
'file_count': len(files), 'weight_count': len(weights),
'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config},
'quantization': quantization,
'checks': ['权重文件存在', '配置JSON及分片引用检查通过'],
'runtime_status': 'NOT_CHECKED',
'runtime_note': '文件已导入,运行能力需接入验证',
}
class ModelImportStore:
def __init__(self, root):
self.root = Path(root)
self.root.mkdir(parents=True, exist_ok=True)
def _records(self):
records = []
for path in self.root.glob('*/import.json'):
if path.parent.name.startswith('.import-'):
continue
try:
records.append(json.loads(path.read_text(encoding='utf-8')))
except (ValueError, OSError):
continue
return sorted(records, key=lambda r: r.get('created_at', ''), reverse=True)
def list(self):
records, paths, fingerprints = [], set(), set()
for record in self._records():
path = record.get('path')
canonical = os.path.normcase(os.path.realpath(path)) if path else None
fingerprint = record.get('fingerprint') if record.get('source') == 'UPLOAD' else None
if (canonical and canonical in paths) or (fingerprint and fingerprint in fingerprints):
continue
if canonical:
paths.add(canonical)
if fingerprint:
fingerprints.add(fingerprint)
records.append(record)
if len(records) == 200:
break
return records
def create(self, owner_id, path=None, upload=None):
if (path is None) == (upload is None):
raise ImportError('请选择一种导入方式')
identity = str(uuid.uuid4())
staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root))
try:
if upload is not None:
if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES:
raise ImportError('请上传2GB以内的ZIP模型包')
archive = staging / 'upload.zip'
with archive.open('wb') as output:
size = 0
for chunk in upload.chunks():
size += len(chunk)
if size > MAX_BYTES:
raise ImportError('模型包超过2GB')
output.write(chunk)
source = staging / 'files'
source.mkdir()
try:
with zipfile.ZipFile(archive) as zipped:
members = zipped.infolist()
if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES:
raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录')
seen = set()
for member in members:
name = PurePosixPath(member.filename)
mode = member.external_attr >> 16
if (name.is_absolute() or '..' in name.parts or '\\' in member.filename
or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1
or member.filename in seen):
raise ImportError('模型包包含不安全的路径或文件')
seen.add(member.filename)
target = source.joinpath(*name.parts)
if member.is_dir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
with zipped.open(member) as incoming, target.open('xb') as output:
shutil.copyfileobj(incoming, output, 1024 * 1024)
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
raise ImportError('模型包损坏或无法解压') from exc
archive.unlink()
# Most model archives contain one enclosing directory.
children = list(source.iterdir())
model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source
info = inspect_model(model_dir)
final_path = self.root / identity / model_dir.relative_to(staging)
source_type = 'UPLOAD'
display_name = Path(upload.name).stem
else:
if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute():
raise ImportError('请输入服务器绝对目录')
model_dir = Path(path).resolve(strict=True)
info = inspect_model(model_dir)
final_path, source_type, display_name = model_dir, 'PATH', model_dir.name
with import_lock(self.root):
existing = self._records()
if source_type == 'PATH':
canonical = os.path.normcase(os.path.realpath(model_dir))
for record in existing:
if record.get('path') and os.path.normcase(os.path.realpath(record['path'])) == canonical:
return {**record, 'already_imported': True}
fingerprint = model_fingerprint(model_dir)
for record in existing:
if (record.get('size_bytes') != info['size_bytes'] or
record.get('file_count') != info['file_count']):
continue
previous_path = record.get('path')
if not previous_path or not Path(previous_path).is_dir():
continue
try:
inspect_model(previous_path)
previous = model_fingerprint(previous_path)
except (OSError, ImportError):
continue
if previous == fingerprint:
return {**record, 'already_imported': True}
record = {
'id': identity, 'name': display_name, 'path': str(final_path),
'source': source_type, 'owner_id': str(owner_id),
'created_at': datetime.now(timezone.utc).isoformat(),
'status': 'IMPORTED', 'fingerprint': fingerprint, **info,
}
(staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8')
staging.rename(self.root / identity)
return record
except (FileNotFoundError, PermissionError) as exc:
raise ImportError('目录不存在或无访问权限') from exc
finally:
if staging.exists():
shutil.rmtree(staging)