UnisKB/apps/models_provider/test_local_imports.py

159 lines
8.1 KiB
Python

"""File import regression tests, temporary files only; no database or model runtimes."""
import io
import json
import tempfile
import unittest
import zipfile
from pathlib import Path
from unittest.mock import patch
from models_provider.local_imports import ModelImportStore, inspect_model, ImportError
class Upload:
def __init__(self, entries):
data = io.BytesIO()
with zipfile.ZipFile(data, 'w') as archive:
for name, content in entries:
archive.writestr(name, content)
self.body = data.getvalue()
self.size, self.name = len(self.body), 'model.zip'
def chunks(self):
yield self.body
class ImportTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.model = self.root / 'model'
self.model.mkdir()
(self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
(self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
self.store = ModelImportStore(self.root / 'imports')
def test_local_path_record_persists_without_copying_or_running(self):
record = self.store.create('owner', path=str(self.model))
self.assertEqual(record['format'], 'HF')
self.assertEqual(record['runtime_status'], 'NOT_CHECKED')
self.assertEqual(record['path'], str(self.model.resolve()))
self.assertEqual(ModelImportStore(self.store.root).list(), [record])
def test_quantized_formats(self):
for method in ('gptq', 'awq'):
(self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}}))
self.assertEqual(inspect_model(self.model)['format'], method.upper())
def test_onnx_gguf_custom(self):
(self.model / 'config.json').unlink()
(self.model / 'model.safetensors').unlink()
for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')):
file = self.model / ('model' + suffix)
file.write_bytes(b'fixture')
(self.model / 'model-manifest.json').write_text('{"version":"1"}')
self.assertEqual(inspect_model(self.model)['format'], expected)
file.unlink()
def test_zip_nested_model_and_persisted_path(self):
record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]))
self.assertTrue(Path(record['path'], 'model.safetensors').exists())
self.assertEqual(record['source'], 'UPLOAD')
def test_traversal_windows_path_rejected(self):
for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'):
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')]))
self.assertEqual(self.store.list(), [])
def test_duplicate_archive_entry_rejected(self):
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')]))
def test_zip_symlink_rejected(self):
data=io.BytesIO()
with zipfile.ZipFile(data,'w') as archive:
info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16)
archive.writestr(info, '../outside')
upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body)
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
def test_size_limit_and_cleanup(self):
upload=Upload([('model.onnx', b'x'*100)])
with patch('models_provider.local_imports.MAX_BYTES', 20):
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
self.assertEqual(list(self.store.root.iterdir()), [])
def test_bad_json_missing_weights_and_unknown_folder_rejected(self):
(self.model / 'config.json').write_text('{broken')
with self.assertRaises(ImportError):inspect_model(self.model)
(self.model / 'config.json').write_text('{"other":1}')
with self.assertRaises(ImportError):inspect_model(self.model)
(self.model / 'model.safetensors').unlink()
with self.assertRaises(ImportError):inspect_model(self.model)
def test_missing_or_escaping_shard_rejected(self):
for name in ('absent.safetensors','../outside.bin'):
(self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}}))
with self.assertRaises(ImportError):inspect_model(self.model)
def test_bad_zip_and_missing_path_leave_no_records(self):
upload=Upload([]);upload.body=b'not zip';upload.size=7
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent'))
self.assertEqual(self.store.list(), [])
def test_exclusive_source(self):
for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}):
with self.assertRaises(ImportError):self.store.create('owner',**kwargs)
def test_repeated_path_and_historical_records_are_deduplicated(self):
first = self.store.create('owner', path=str(self.model))
repeated = self.store.create('owner', path=str(self.model / '..' / 'model'))
self.assertEqual(repeated['id'], first['id'])
self.assertTrue(repeated['already_imported'])
legacy = self.store.root / 'legacy'
legacy.mkdir()
(legacy / 'import.json').write_text(json.dumps({**first, 'id': 'legacy', 'created_at': '2000'}))
self.assertEqual([row['id'] for row in self.store.list()], [first['id']])
def test_zip_content_deduplicates_and_distinct_content_is_retained(self):
files = [('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]
first = self.store.create('owner', upload=Upload(files))
second = self.store.create('owner', upload=Upload(list(reversed(files))))
self.assertEqual(first['id'], second['id'])
self.assertTrue(second['already_imported'])
different = self.store.create('owner', upload=Upload([
('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights2')]))
self.assertNotEqual(first['id'], different['id'])
self.assertEqual(len(self.store.list()), 2)
def test_different_directory_and_zip_with_same_files_reuse_record(self):
first = self.store.create('owner', path=str(self.model))
copy = self.root / 'copy'
import shutil
shutil.copytree(self.model, copy)
second = self.store.create('owner', path=str(copy))
self.assertEqual(first['id'], second['id'])
zipped = Upload([('repo/config.json', (copy / 'config.json').read_bytes()),
('repo/model.safetensors', (copy / 'model.safetensors').read_bytes())])
self.assertEqual(self.store.create('owner', upload=zipped)['id'], first['id'])
def test_concurrent_imports_create_one_record(self):
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(pool.map(lambda _: self.store.create('owner', path=str(self.model)), range(4)))
self.assertEqual(len({row['id'] for row in results}), 1)
self.assertEqual(len(self.store._records()), 1)
def test_invalid_legacy_directory_does_not_block_import(self):
first = self.store.create('owner', path=str(self.model))
config = self.model / 'config.json'
config.write_text('x' * len(config.read_text()))
replacement = self.root / 'replacement'
replacement.mkdir()
(replacement / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
(replacement / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
self.assertNotEqual(self.store.create('owner', path=str(replacement))['id'], first['id'])
if __name__ == '__main__':unittest.main()