200 lines
8.8 KiB
Python
200 lines
8.8 KiB
Python
import contextlib
|
|
import json
|
|
import threading
|
|
import weakref
|
|
import unittest
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from common.model_runtime import ModelRuntime, loading_policy
|
|
from common.model_runtime_client import signed_request, verify_request
|
|
|
|
|
|
class FakeAdapter:
|
|
def __init__(self):
|
|
self.loaded, self.unloaded = [], []
|
|
self.native_ready = True
|
|
self.fail = False
|
|
self.block = None
|
|
|
|
def load(self, spec):
|
|
if self.block:
|
|
self.block.wait(3)
|
|
if self.fail:
|
|
raise ValueError('private error details')
|
|
self.loaded.append(spec['id'])
|
|
self.native_ready = True
|
|
return SimpleNamespace(), {'device': 'cpu'}
|
|
|
|
def unload(self, spec, value):
|
|
self.unloaded.append(spec['id'])
|
|
self.native_ready = False
|
|
|
|
def probe(self, spec):
|
|
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
|
|
|
|
def collect(self):
|
|
pass
|
|
|
|
def error(self, exc):
|
|
return 'load failed'
|
|
|
|
|
|
class RuntimeTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.now = 0
|
|
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
|
|
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
|
|
for key, priority in [('low', 10), ('high', 90)]}
|
|
self.adapter = FakeAdapter()
|
|
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
|
|
self.adapter, lambda: self.now)
|
|
|
|
def test_no_load_until_first_inference_and_then_reuses_weights(self):
|
|
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
|
|
self.assertEqual(self.adapter.loaded, [])
|
|
with self.manager.inference('low') as first:
|
|
self.assertIsNotNone(first)
|
|
with self.manager.inference('low') as second:
|
|
self.assertIs(first, second)
|
|
self.assertEqual(self.adapter.loaded, ['low'])
|
|
|
|
def test_startup_preloads_highest_priority_first(self):
|
|
self.specs['low']['policy']['strategy'] = 'PRELOAD'
|
|
self.specs['high']['policy']['strategy'] = 'PRELOAD'
|
|
self.manager.preload(['low', 'high'])
|
|
for key in ['low', 'high']:
|
|
self.manager.entries[key]['future'].result(3)
|
|
self.assertEqual(self.adapter.loaded, ['high', 'low'])
|
|
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
|
|
|
|
def test_concurrent_first_calls_only_load_once(self):
|
|
self.adapter.block = threading.Event()
|
|
with ThreadPoolExecutor(max_workers=6) as pool:
|
|
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
|
|
self.adapter.block.set()
|
|
tokens = [future.result(3) for future in futures]
|
|
self.assertEqual(len(set(tokens)), 6)
|
|
self.assertEqual(self.adapter.loaded, ['low'])
|
|
self.assertEqual(self.manager.status('low')['active_requests'], 6)
|
|
for token in tokens:
|
|
self.manager.release('low', token)
|
|
|
|
def test_in_use_models_cannot_be_unloaded(self):
|
|
token = self.manager.acquire('low')
|
|
with self.assertRaises(ValueError):
|
|
self.manager.unload('low')
|
|
self.manager.release('low', token)
|
|
self.manager.unload('low')
|
|
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
|
|
|
|
def test_unload_releases_the_last_loader_reference(self):
|
|
class Weights:
|
|
pass
|
|
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
|
|
token = self.manager.acquire('low')
|
|
pointer = weakref.ref(self.manager.entries['low']['value'])
|
|
self.manager.release('low', token)
|
|
self.manager.unload('low')
|
|
self.assertIsNone(pointer())
|
|
|
|
def test_idle_cleanup_respects_active_and_preloaded_models(self):
|
|
active = self.manager.acquire('low')
|
|
self.specs['high']['policy']['strategy'] = 'PRELOAD'
|
|
self.manager.preload(['high'])
|
|
self.manager.entries['high']['future'].result(3)
|
|
self.now = 100
|
|
self.manager.sweep()
|
|
self.assertEqual(self.adapter.unloaded, [])
|
|
self.manager.release('low', active)
|
|
self.now = 131
|
|
self.manager.sweep()
|
|
self.assertEqual(self.adapter.unloaded, ['low'])
|
|
self.assertEqual(self.manager.status('high')['status'], 'READY')
|
|
|
|
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
|
|
self.adapter.fail = True
|
|
with self.assertRaisesRegex(ValueError, '^load failed$'):
|
|
self.manager.acquire('low')
|
|
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
|
|
self.adapter.fail = False
|
|
with self.manager.inference('low'):
|
|
pass
|
|
self.assertEqual(self.manager.status('low')['status'], 'READY')
|
|
|
|
def test_changed_model_config_never_reuses_old_weights(self):
|
|
token = self.manager.acquire('low')
|
|
self.specs['low']['fingerprint'] = 'v2'
|
|
with self.assertRaises(ValueError):
|
|
self.manager.acquire('low')
|
|
self.manager.release('low', token)
|
|
with self.manager.inference('low'):
|
|
pass
|
|
self.assertEqual(self.adapter.loaded, ['low', 'low'])
|
|
self.assertEqual(self.adapter.unloaded, ['low'])
|
|
|
|
def test_native_expiration_requires_loading_again(self):
|
|
self.specs['low']['driver'] = 'ollama'
|
|
token = self.manager.acquire('low')
|
|
self.manager.release('low', token)
|
|
self.adapter.native_ready = False
|
|
token = self.manager.acquire('low')
|
|
self.manager.release('low', token)
|
|
self.assertEqual(self.adapter.loaded, ['low', 'low'])
|
|
|
|
def test_policy_validation(self):
|
|
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
|
|
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
|
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
|
loading_policy(value)
|
|
|
|
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
|
|
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
|
|
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
|
|
self.assertEqual(verify_request(body, signature)['operation'], 'load')
|
|
with self.assertRaises(ValueError):
|
|
verify_request(body.replace(b'load', b'unload'), signature)
|
|
with self.assertRaises(ValueError):
|
|
verify_request(body, '')
|
|
with patch('common.model_runtime_client.time.time', return_value=1):
|
|
body, signature = signed_request({'operation': 'load'})
|
|
with self.assertRaises(ValueError):
|
|
verify_request(body, signature)
|
|
|
|
|
|
class ProviderHookTests(unittest.TestCase):
|
|
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
|
|
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
|
|
events = []
|
|
@contextlib.contextmanager
|
|
def lease(model_id):
|
|
events.append(('acquire', model_id))
|
|
try:
|
|
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
|
|
finally:
|
|
events.append(('release', model_id))
|
|
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
|
|
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
|
|
self.assertEqual(model.embed_query('text'), [1., 2.])
|
|
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
|
|
self.assertEqual(model._default_params['keep_alive'], -1)
|
|
|
|
def test_llm_stream_releases_lease_when_generator_is_closed(self):
|
|
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
|
|
from langchain_ollama import ChatOllama
|
|
events = []
|
|
@contextlib.contextmanager
|
|
def lease(model_id):
|
|
events.append('acquire')
|
|
try:
|
|
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
|
|
finally:
|
|
events.append('release')
|
|
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
|
|
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
|
|
stream = model._create_chat_stream([])
|
|
next(stream); stream.close()
|
|
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
|
|
self.assertEqual(events, ['acquire', 'release'])
|