203 lines
8.8 KiB
Python
203 lines
8.8 KiB
Python
"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
|
||
import contextlib
|
||
import itertools
|
||
import queue
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from concurrent.futures import Future
|
||
|
||
|
||
def loading_policy(value=None):
|
||
value = {} if value is None else value
|
||
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
|
||
raise ValueError('加载策略无效')
|
||
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
|
||
'idle_seconds': value.get('idle_seconds', 300)}
|
||
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
|
||
raise ValueError('加载方式无效')
|
||
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
|
||
raise ValueError('加载优先级须为0~100的整数')
|
||
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
|
||
raise ValueError('空闲时间须为30~86400秒')
|
||
return result
|
||
|
||
|
||
class ModelRuntime:
|
||
def __init__(self, specs, adapter, clock=time.monotonic):
|
||
self.specs, self.adapter, self.clock = specs, adapter, clock
|
||
self.entries = {}
|
||
self.lock = threading.RLock()
|
||
self.queue = queue.PriorityQueue()
|
||
self.sequence = itertools.count()
|
||
self.load_sequence = itertools.count(1)
|
||
self.worker = None
|
||
|
||
def _entry(self, model_id):
|
||
model_id = str(model_id)
|
||
spec = self.specs(model_id)
|
||
spec['policy'] = loading_policy(spec.get('policy'))
|
||
entry = self.entries.get(model_id)
|
||
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
|
||
if entry['leases'] or entry['state'] == 'LOADING':
|
||
raise ValueError('模型配置已变更,请等待当前请求结束')
|
||
self._discard(entry)
|
||
entry = None
|
||
if entry is None:
|
||
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
|
||
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
|
||
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
|
||
'load_order': None}
|
||
self.entries[model_id] = entry
|
||
previous = entry['spec']['policy']
|
||
entry['spec'] = spec
|
||
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
|
||
entry['generation'] += 1
|
||
self._put(entry)
|
||
return entry
|
||
|
||
def _put(self, entry):
|
||
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
|
||
|
||
def start(self):
|
||
with self.lock:
|
||
if self.worker is None:
|
||
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
|
||
self.worker.start()
|
||
|
||
def submit(self, model_id, start=True):
|
||
with self.lock:
|
||
entry = self._entry(model_id)
|
||
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
|
||
entry.update(state='UNLOADED', value=None, info={})
|
||
if entry['state'] == 'READY':
|
||
return entry
|
||
if entry['state'] not in ('LOADING', 'QUEUED'):
|
||
entry.update(state='QUEUED', error='', future=Future())
|
||
entry['generation'] += 1
|
||
self._put(entry)
|
||
if start:
|
||
self.start()
|
||
return entry
|
||
|
||
def preload(self, model_ids):
|
||
# Enqueue the complete startup batch before choosing the highest-priority model.
|
||
with self.lock:
|
||
for model_id in model_ids:
|
||
self.submit(model_id, start=False)
|
||
self.start()
|
||
|
||
def _work(self):
|
||
while True:
|
||
_, order, model_id, generation = self.queue.get()
|
||
try:
|
||
with self.lock:
|
||
entry = self.entries.get(model_id)
|
||
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
|
||
continue
|
||
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
|
||
spec = entry['spec']
|
||
future = entry['future']
|
||
value, info = self.adapter.load(spec)
|
||
with self.lock:
|
||
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
|
||
future.set_result(True)
|
||
del value # The loader thread must not keep the last model alive after eviction.
|
||
except Exception as exc:
|
||
with self.lock:
|
||
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
|
||
if not entry['future'].done():
|
||
entry['future'].set_exception(ValueError(entry['error']))
|
||
finally:
|
||
self.queue.task_done()
|
||
|
||
def acquire(self, model_id, remote=False, timeout=600):
|
||
with self.lock:
|
||
entry = self.submit(model_id)
|
||
token = uuid.uuid4().hex
|
||
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
|
||
future = entry['future'] if entry['state'] != 'READY' else None
|
||
try:
|
||
if future:
|
||
future.result(timeout=timeout)
|
||
return token
|
||
except Exception:
|
||
self.release(model_id, token)
|
||
raise
|
||
|
||
def release(self, model_id, token):
|
||
with self.lock:
|
||
entry = self.entries.get(str(model_id))
|
||
if entry:
|
||
entry['leases'].pop(token, None)
|
||
entry['last_used'] = self.clock()
|
||
|
||
@contextlib.contextmanager
|
||
def inference(self, model_id):
|
||
token = self.acquire(model_id)
|
||
try:
|
||
with self.lock:
|
||
value = self.entries[str(model_id)]['value']
|
||
yield value
|
||
finally:
|
||
self.release(model_id, token)
|
||
|
||
def _discard(self, entry):
|
||
if entry['value'] is not None:
|
||
value = entry['value']
|
||
self.adapter.unload(entry['spec'], value)
|
||
entry['value'] = None
|
||
del value
|
||
self.adapter.collect()
|
||
entry.update(state='UNLOADED', loaded_at=None, info={})
|
||
|
||
def unload(self, model_id):
|
||
with self.lock:
|
||
entry = self._entry(model_id)
|
||
self._expire_leases(entry)
|
||
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
|
||
raise ValueError('模型正在使用或加载,请稍后再试')
|
||
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
|
||
entry['value'] = True
|
||
self._discard(entry)
|
||
|
||
def configure(self, model_id):
|
||
with self.lock:
|
||
entry = self._entry(model_id)
|
||
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
|
||
if warm_again:
|
||
entry.update(state='UNLOADED', value=None)
|
||
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
|
||
self.submit(model_id)
|
||
|
||
def _expire_leases(self, entry):
|
||
now = self.clock()
|
||
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
|
||
|
||
def sweep(self):
|
||
with self.lock:
|
||
for entry in list(self.entries.values()):
|
||
self._expire_leases(entry)
|
||
policy = entry['spec']['policy']
|
||
# Ollama owns its idle timer; do not unload models used by other clients.
|
||
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
|
||
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
|
||
self.clock() - entry['last_used'] >= policy['idle_seconds']):
|
||
self._discard(entry)
|
||
|
||
def status(self, model_id):
|
||
with self.lock:
|
||
entry = self._entry(model_id)
|
||
self._expire_leases(entry)
|
||
# Reconcile native residency, including expiration outside this process.
|
||
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
|
||
info = self.adapter.probe(entry['spec'])
|
||
if info:
|
||
entry.update(state='READY', info=info, value=True)
|
||
elif entry['state'] == 'READY':
|
||
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
|
||
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
|
||
'active_requests': len(entry['leases']), 'error': entry['error'],
|
||
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
|
||
'load_order': entry['load_order'], **entry['info']}
|