UnisKB/apps/models_provider/test_model_runtime.py

200 lines
8.8 KiB
Python

import contextlib
import json
import threading
import weakref
import unittest
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
from unittest.mock import patch
from common.model_runtime import ModelRuntime, loading_policy
from common.model_runtime_client import signed_request, verify_request
class FakeAdapter:
def __init__(self):
self.loaded, self.unloaded = [], []
self.native_ready = True
self.fail = False
self.block = None
def load(self, spec):
if self.block:
self.block.wait(3)
if self.fail:
raise ValueError('private error details')
self.loaded.append(spec['id'])
self.native_ready = True
return SimpleNamespace(), {'device': 'cpu'}
def unload(self, spec, value):
self.unloaded.append(spec['id'])
self.native_ready = False
def probe(self, spec):
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
def collect(self):
pass
def error(self, exc):
return 'load failed'
class RuntimeTests(unittest.TestCase):
def setUp(self):
self.now = 0
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
for key, priority in [('low', 10), ('high', 90)]}
self.adapter = FakeAdapter()
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
self.adapter, lambda: self.now)
def test_no_load_until_first_inference_and_then_reuses_weights(self):
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
self.assertEqual(self.adapter.loaded, [])
with self.manager.inference('low') as first:
self.assertIsNotNone(first)
with self.manager.inference('low') as second:
self.assertIs(first, second)
self.assertEqual(self.adapter.loaded, ['low'])
def test_startup_preloads_highest_priority_first(self):
self.specs['low']['policy']['strategy'] = 'PRELOAD'
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['low', 'high'])
for key in ['low', 'high']:
self.manager.entries[key]['future'].result(3)
self.assertEqual(self.adapter.loaded, ['high', 'low'])
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
def test_concurrent_first_calls_only_load_once(self):
self.adapter.block = threading.Event()
with ThreadPoolExecutor(max_workers=6) as pool:
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
self.adapter.block.set()
tokens = [future.result(3) for future in futures]
self.assertEqual(len(set(tokens)), 6)
self.assertEqual(self.adapter.loaded, ['low'])
self.assertEqual(self.manager.status('low')['active_requests'], 6)
for token in tokens:
self.manager.release('low', token)
def test_in_use_models_cannot_be_unloaded(self):
token = self.manager.acquire('low')
with self.assertRaises(ValueError):
self.manager.unload('low')
self.manager.release('low', token)
self.manager.unload('low')
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
def test_unload_releases_the_last_loader_reference(self):
class Weights:
pass
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
token = self.manager.acquire('low')
pointer = weakref.ref(self.manager.entries['low']['value'])
self.manager.release('low', token)
self.manager.unload('low')
self.assertIsNone(pointer())
def test_idle_cleanup_respects_active_and_preloaded_models(self):
active = self.manager.acquire('low')
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['high'])
self.manager.entries['high']['future'].result(3)
self.now = 100
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, [])
self.manager.release('low', active)
self.now = 131
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, ['low'])
self.assertEqual(self.manager.status('high')['status'], 'READY')
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
self.adapter.fail = True
with self.assertRaisesRegex(ValueError, '^load failed$'):
self.manager.acquire('low')
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
self.adapter.fail = False
with self.manager.inference('low'):
pass
self.assertEqual(self.manager.status('low')['status'], 'READY')
def test_changed_model_config_never_reuses_old_weights(self):
token = self.manager.acquire('low')
self.specs['low']['fingerprint'] = 'v2'
with self.assertRaises(ValueError):
self.manager.acquire('low')
self.manager.release('low', token)
with self.manager.inference('low'):
pass
self.assertEqual(self.adapter.loaded, ['low', 'low'])
self.assertEqual(self.adapter.unloaded, ['low'])
def test_native_expiration_requires_loading_again(self):
self.specs['low']['driver'] = 'ollama'
token = self.manager.acquire('low')
self.manager.release('low', token)
self.adapter.native_ready = False
token = self.manager.acquire('low')
self.manager.release('low', token)
self.assertEqual(self.adapter.loaded, ['low', 'low'])
def test_policy_validation(self):
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
with self.subTest(value=value), self.assertRaises(ValueError):
loading_policy(value)
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
self.assertEqual(verify_request(body, signature)['operation'], 'load')
with self.assertRaises(ValueError):
verify_request(body.replace(b'load', b'unload'), signature)
with self.assertRaises(ValueError):
verify_request(body, '')
with patch('common.model_runtime_client.time.time', return_value=1):
body, signature = signed_request({'operation': 'load'})
with self.assertRaises(ValueError):
verify_request(body, signature)
class ProviderHookTests(unittest.TestCase):
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
events = []
@contextlib.contextmanager
def lease(model_id):
events.append(('acquire', model_id))
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append(('release', model_id))
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
self.assertEqual(model.embed_query('text'), [1., 2.])
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
self.assertEqual(model._default_params['keep_alive'], -1)
def test_llm_stream_releases_lease_when_generator_is_closed(self):
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
from langchain_ollama import ChatOllama
events = []
@contextlib.contextmanager
def lease(model_id):
events.append('acquire')
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append('release')
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
stream = model._create_chat_stream([])
next(stream); stream.close()
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
self.assertEqual(events, ['acquire', 'release'])