import contextlib import json import threading import weakref import unittest from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import patch from common.model_runtime import ModelRuntime, loading_policy from common.model_runtime_client import signed_request, verify_request class FakeAdapter: def __init__(self): self.loaded, self.unloaded = [], [] self.native_ready = True self.fail = False self.block = None def load(self, spec): if self.block: self.block.wait(3) if self.fail: raise ValueError('private error details') self.loaded.append(spec['id']) self.native_ready = True return SimpleNamespace(), {'device': 'cpu'} def unload(self, spec, value): self.unloaded.append(spec['id']) self.native_ready = False def probe(self, spec): return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None def collect(self): pass def error(self, exc): return 'load failed' class RuntimeTests(unittest.TestCase): def setUp(self): self.now = 0 self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1', 'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}} for key, priority in [('low', 10), ('high', 90)]} self.adapter = FakeAdapter() self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])}, self.adapter, lambda: self.now) def test_no_load_until_first_inference_and_then_reuses_weights(self): self.assertEqual(self.manager.status('low')['status'], 'UNLOADED') self.assertEqual(self.adapter.loaded, []) with self.manager.inference('low') as first: self.assertIsNotNone(first) with self.manager.inference('low') as second: self.assertIs(first, second) self.assertEqual(self.adapter.loaded, ['low']) def test_startup_preloads_highest_priority_first(self): self.specs['low']['policy']['strategy'] = 'PRELOAD' self.specs['high']['policy']['strategy'] = 'PRELOAD' self.manager.preload(['low', 'high']) for key in ['low', 'high']: self.manager.entries[key]['future'].result(3) self.assertEqual(self.adapter.loaded, ['high', 'low']) self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order']) def test_concurrent_first_calls_only_load_once(self): self.adapter.block = threading.Event() with ThreadPoolExecutor(max_workers=6) as pool: futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)] self.adapter.block.set() tokens = [future.result(3) for future in futures] self.assertEqual(len(set(tokens)), 6) self.assertEqual(self.adapter.loaded, ['low']) self.assertEqual(self.manager.status('low')['active_requests'], 6) for token in tokens: self.manager.release('low', token) def test_in_use_models_cannot_be_unloaded(self): token = self.manager.acquire('low') with self.assertRaises(ValueError): self.manager.unload('low') self.manager.release('low', token) self.manager.unload('low') self.assertEqual(self.manager.status('low')['status'], 'UNLOADED') def test_unload_releases_the_last_loader_reference(self): class Weights: pass with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})): token = self.manager.acquire('low') pointer = weakref.ref(self.manager.entries['low']['value']) self.manager.release('low', token) self.manager.unload('low') self.assertIsNone(pointer()) def test_idle_cleanup_respects_active_and_preloaded_models(self): active = self.manager.acquire('low') self.specs['high']['policy']['strategy'] = 'PRELOAD' self.manager.preload(['high']) self.manager.entries['high']['future'].result(3) self.now = 100 self.manager.sweep() self.assertEqual(self.adapter.unloaded, []) self.manager.release('low', active) self.now = 131 self.manager.sweep() self.assertEqual(self.adapter.unloaded, ['low']) self.assertEqual(self.manager.status('high')['status'], 'READY') def test_failed_load_can_be_retried_and_private_errors_are_hidden(self): self.adapter.fail = True with self.assertRaisesRegex(ValueError, '^load failed$'): self.manager.acquire('low') self.assertEqual(self.manager.status('low')['status'], 'FAILED') self.adapter.fail = False with self.manager.inference('low'): pass self.assertEqual(self.manager.status('low')['status'], 'READY') def test_changed_model_config_never_reuses_old_weights(self): token = self.manager.acquire('low') self.specs['low']['fingerprint'] = 'v2' with self.assertRaises(ValueError): self.manager.acquire('low') self.manager.release('low', token) with self.manager.inference('low'): pass self.assertEqual(self.adapter.loaded, ['low', 'low']) self.assertEqual(self.adapter.unloaded, ['low']) def test_native_expiration_requires_loading_again(self): self.specs['low']['driver'] = 'ollama' token = self.manager.acquire('low') self.manager.release('low', token) self.adapter.native_ready = False token = self.manager.acquire('low') self.manager.release('low', token) self.assertEqual(self.adapter.loaded, ['low', 'low']) def test_policy_validation(self): for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'}, {'idle_seconds': 0}, {'engine_url': 'http://example.com'}): with self.subTest(value=value), self.assertRaises(ValueError): loading_policy(value) def test_internal_rpc_rejects_forged_or_expired_payloads(self): with patch('common.model_runtime_client._secret', return_value=b'test-only-key'): body, signature = signed_request({'operation': 'load', 'model_id': 'test'}) self.assertEqual(verify_request(body, signature)['operation'], 'load') with self.assertRaises(ValueError): verify_request(body.replace(b'load', b'unload'), signature) with self.assertRaises(ValueError): verify_request(body, '') with patch('common.model_runtime_client.time.time', return_value=1): body, signature = signed_request({'operation': 'load'}) with self.assertRaises(ValueError): verify_request(body, signature) class ProviderHookTests(unittest.TestCase): def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self): from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding events = [] @contextlib.contextmanager def lease(model_id): events.append(('acquire', model_id)) try: yield {'strategy': 'PRELOAD', 'idle_seconds': 300} finally: events.append(('release', model_id)) model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource') with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]): self.assertEqual(model.embed_query('text'), [1., 2.]) self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')]) self.assertEqual(model._default_params['keep_alive'], -1) def test_llm_stream_releases_lease_when_generator_is_closed(self): from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel from langchain_ollama import ChatOllama events = [] @contextlib.contextmanager def lease(model_id): events.append('acquire') try: yield {'strategy': 'PRELOAD', 'idle_seconds': 300} finally: events.append('release') model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource') with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native: stream = model._create_chat_stream([]) next(stream); stream.close() self.assertEqual(native.call_args.kwargs['keep_alive'], -1) self.assertEqual(events, ['acquire', 'release'])