159 lines
8.1 KiB
Python
159 lines
8.1 KiB
Python
"""File import regression tests, temporary files only; no database or model runtimes."""
|
|
import io
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
import zipfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from models_provider.local_imports import ModelImportStore, inspect_model, ImportError
|
|
|
|
|
|
class Upload:
|
|
def __init__(self, entries):
|
|
data = io.BytesIO()
|
|
with zipfile.ZipFile(data, 'w') as archive:
|
|
for name, content in entries:
|
|
archive.writestr(name, content)
|
|
self.body = data.getvalue()
|
|
self.size, self.name = len(self.body), 'model.zip'
|
|
|
|
def chunks(self):
|
|
yield self.body
|
|
|
|
|
|
class ImportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.model = self.root / 'model'
|
|
self.model.mkdir()
|
|
(self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
|
|
(self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
|
|
self.store = ModelImportStore(self.root / 'imports')
|
|
|
|
def test_local_path_record_persists_without_copying_or_running(self):
|
|
record = self.store.create('owner', path=str(self.model))
|
|
self.assertEqual(record['format'], 'HF')
|
|
self.assertEqual(record['runtime_status'], 'NOT_CHECKED')
|
|
self.assertEqual(record['path'], str(self.model.resolve()))
|
|
self.assertEqual(ModelImportStore(self.store.root).list(), [record])
|
|
|
|
def test_quantized_formats(self):
|
|
for method in ('gptq', 'awq'):
|
|
(self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}}))
|
|
self.assertEqual(inspect_model(self.model)['format'], method.upper())
|
|
|
|
def test_onnx_gguf_custom(self):
|
|
(self.model / 'config.json').unlink()
|
|
(self.model / 'model.safetensors').unlink()
|
|
for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')):
|
|
file = self.model / ('model' + suffix)
|
|
file.write_bytes(b'fixture')
|
|
(self.model / 'model-manifest.json').write_text('{"version":"1"}')
|
|
self.assertEqual(inspect_model(self.model)['format'], expected)
|
|
file.unlink()
|
|
|
|
def test_zip_nested_model_and_persisted_path(self):
|
|
record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]))
|
|
self.assertTrue(Path(record['path'], 'model.safetensors').exists())
|
|
self.assertEqual(record['source'], 'UPLOAD')
|
|
|
|
def test_traversal_windows_path_rejected(self):
|
|
for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'):
|
|
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')]))
|
|
self.assertEqual(self.store.list(), [])
|
|
|
|
def test_duplicate_archive_entry_rejected(self):
|
|
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')]))
|
|
|
|
def test_zip_symlink_rejected(self):
|
|
data=io.BytesIO()
|
|
with zipfile.ZipFile(data,'w') as archive:
|
|
info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16)
|
|
archive.writestr(info, '../outside')
|
|
upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body)
|
|
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
|
|
|
def test_size_limit_and_cleanup(self):
|
|
upload=Upload([('model.onnx', b'x'*100)])
|
|
with patch('models_provider.local_imports.MAX_BYTES', 20):
|
|
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
|
self.assertEqual(list(self.store.root.iterdir()), [])
|
|
|
|
def test_bad_json_missing_weights_and_unknown_folder_rejected(self):
|
|
(self.model / 'config.json').write_text('{broken')
|
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
|
(self.model / 'config.json').write_text('{"other":1}')
|
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
|
(self.model / 'model.safetensors').unlink()
|
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
|
|
|
def test_missing_or_escaping_shard_rejected(self):
|
|
for name in ('absent.safetensors','../outside.bin'):
|
|
(self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}}))
|
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
|
|
|
def test_bad_zip_and_missing_path_leave_no_records(self):
|
|
upload=Upload([]);upload.body=b'not zip';upload.size=7
|
|
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
|
with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent'))
|
|
self.assertEqual(self.store.list(), [])
|
|
|
|
def test_exclusive_source(self):
|
|
for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}):
|
|
with self.assertRaises(ImportError):self.store.create('owner',**kwargs)
|
|
|
|
def test_repeated_path_and_historical_records_are_deduplicated(self):
|
|
first = self.store.create('owner', path=str(self.model))
|
|
repeated = self.store.create('owner', path=str(self.model / '..' / 'model'))
|
|
self.assertEqual(repeated['id'], first['id'])
|
|
self.assertTrue(repeated['already_imported'])
|
|
legacy = self.store.root / 'legacy'
|
|
legacy.mkdir()
|
|
(legacy / 'import.json').write_text(json.dumps({**first, 'id': 'legacy', 'created_at': '2000'}))
|
|
self.assertEqual([row['id'] for row in self.store.list()], [first['id']])
|
|
|
|
def test_zip_content_deduplicates_and_distinct_content_is_retained(self):
|
|
files = [('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]
|
|
first = self.store.create('owner', upload=Upload(files))
|
|
second = self.store.create('owner', upload=Upload(list(reversed(files))))
|
|
self.assertEqual(first['id'], second['id'])
|
|
self.assertTrue(second['already_imported'])
|
|
different = self.store.create('owner', upload=Upload([
|
|
('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights2')]))
|
|
self.assertNotEqual(first['id'], different['id'])
|
|
self.assertEqual(len(self.store.list()), 2)
|
|
|
|
def test_different_directory_and_zip_with_same_files_reuse_record(self):
|
|
first = self.store.create('owner', path=str(self.model))
|
|
copy = self.root / 'copy'
|
|
import shutil
|
|
shutil.copytree(self.model, copy)
|
|
second = self.store.create('owner', path=str(copy))
|
|
self.assertEqual(first['id'], second['id'])
|
|
zipped = Upload([('repo/config.json', (copy / 'config.json').read_bytes()),
|
|
('repo/model.safetensors', (copy / 'model.safetensors').read_bytes())])
|
|
self.assertEqual(self.store.create('owner', upload=zipped)['id'], first['id'])
|
|
|
|
def test_concurrent_imports_create_one_record(self):
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
results = list(pool.map(lambda _: self.store.create('owner', path=str(self.model)), range(4)))
|
|
self.assertEqual(len({row['id'] for row in results}), 1)
|
|
self.assertEqual(len(self.store._records()), 1)
|
|
|
|
def test_invalid_legacy_directory_does_not_block_import(self):
|
|
first = self.store.create('owner', path=str(self.model))
|
|
config = self.model / 'config.json'
|
|
config.write_text('x' * len(config.read_text()))
|
|
replacement = self.root / 'replacement'
|
|
replacement.mkdir()
|
|
(replacement / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
|
|
(replacement / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
|
|
self.assertNotEqual(self.store.create('owner', path=str(replacement))['id'], first['id'])
|
|
|
|
|
|
if __name__ == '__main__':unittest.main()
|