UnisKB/apps/common/model_runtime.py

203 lines
8.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
import contextlib
import itertools
import queue
import threading
import time
import uuid
from concurrent.futures import Future
def loading_policy(value=None):
value = {} if value is None else value
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
raise ValueError('加载策略无效')
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
'idle_seconds': value.get('idle_seconds', 300)}
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
raise ValueError('加载方式无效')
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
raise ValueError('加载优先级须为0~100的整数')
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
raise ValueError('空闲时间须为30~86400秒')
return result
class ModelRuntime:
def __init__(self, specs, adapter, clock=time.monotonic):
self.specs, self.adapter, self.clock = specs, adapter, clock
self.entries = {}
self.lock = threading.RLock()
self.queue = queue.PriorityQueue()
self.sequence = itertools.count()
self.load_sequence = itertools.count(1)
self.worker = None
def _entry(self, model_id):
model_id = str(model_id)
spec = self.specs(model_id)
spec['policy'] = loading_policy(spec.get('policy'))
entry = self.entries.get(model_id)
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
if entry['leases'] or entry['state'] == 'LOADING':
raise ValueError('模型配置已变更,请等待当前请求结束')
self._discard(entry)
entry = None
if entry is None:
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
'load_order': None}
self.entries[model_id] = entry
previous = entry['spec']['policy']
entry['spec'] = spec
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
entry['generation'] += 1
self._put(entry)
return entry
def _put(self, entry):
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
def start(self):
with self.lock:
if self.worker is None:
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
self.worker.start()
def submit(self, model_id, start=True):
with self.lock:
entry = self._entry(model_id)
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
entry.update(state='UNLOADED', value=None, info={})
if entry['state'] == 'READY':
return entry
if entry['state'] not in ('LOADING', 'QUEUED'):
entry.update(state='QUEUED', error='', future=Future())
entry['generation'] += 1
self._put(entry)
if start:
self.start()
return entry
def preload(self, model_ids):
# Enqueue the complete startup batch before choosing the highest-priority model.
with self.lock:
for model_id in model_ids:
self.submit(model_id, start=False)
self.start()
def _work(self):
while True:
_, order, model_id, generation = self.queue.get()
try:
with self.lock:
entry = self.entries.get(model_id)
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
continue
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
spec = entry['spec']
future = entry['future']
value, info = self.adapter.load(spec)
with self.lock:
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
future.set_result(True)
del value # The loader thread must not keep the last model alive after eviction.
except Exception as exc:
with self.lock:
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
if not entry['future'].done():
entry['future'].set_exception(ValueError(entry['error']))
finally:
self.queue.task_done()
def acquire(self, model_id, remote=False, timeout=600):
with self.lock:
entry = self.submit(model_id)
token = uuid.uuid4().hex
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
future = entry['future'] if entry['state'] != 'READY' else None
try:
if future:
future.result(timeout=timeout)
return token
except Exception:
self.release(model_id, token)
raise
def release(self, model_id, token):
with self.lock:
entry = self.entries.get(str(model_id))
if entry:
entry['leases'].pop(token, None)
entry['last_used'] = self.clock()
@contextlib.contextmanager
def inference(self, model_id):
token = self.acquire(model_id)
try:
with self.lock:
value = self.entries[str(model_id)]['value']
yield value
finally:
self.release(model_id, token)
def _discard(self, entry):
if entry['value'] is not None:
value = entry['value']
self.adapter.unload(entry['spec'], value)
entry['value'] = None
del value
self.adapter.collect()
entry.update(state='UNLOADED', loaded_at=None, info={})
def unload(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
raise ValueError('模型正在使用或加载,请稍后再试')
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
entry['value'] = True
self._discard(entry)
def configure(self, model_id):
with self.lock:
entry = self._entry(model_id)
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
if warm_again:
entry.update(state='UNLOADED', value=None)
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
self.submit(model_id)
def _expire_leases(self, entry):
now = self.clock()
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
def sweep(self):
with self.lock:
for entry in list(self.entries.values()):
self._expire_leases(entry)
policy = entry['spec']['policy']
# Ollama owns its idle timer; do not unload models used by other clients.
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
self.clock() - entry['last_used'] >= policy['idle_seconds']):
self._discard(entry)
def status(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
# Reconcile native residency, including expiration outside this process.
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
info = self.adapter.probe(entry['spec'])
if info:
entry.update(state='READY', info=info, value=True)
elif entry['state'] == 'READY':
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
'active_requests': len(entry['leases']), 'error': entry['error'],
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
'load_order': entry['load_order'], **entry['info']}