"""Single-owner, priority-ordered weight loading. No engine or framework dependency.""" import contextlib import itertools import queue import threading import time import uuid from concurrent.futures import Future def loading_policy(value=None): value = {} if value is None else value if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}: raise ValueError('加载策略无效') result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50), 'idle_seconds': value.get('idle_seconds', 300)} if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'): raise ValueError('加载方式无效') if type(result['priority']) is not int or not 0 <= result['priority'] <= 100: raise ValueError('加载优先级须为0~100的整数') if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400: raise ValueError('空闲时间须为30~86400秒') return result class ModelRuntime: def __init__(self, specs, adapter, clock=time.monotonic): self.specs, self.adapter, self.clock = specs, adapter, clock self.entries = {} self.lock = threading.RLock() self.queue = queue.PriorityQueue() self.sequence = itertools.count() self.load_sequence = itertools.count(1) self.worker = None def _entry(self, model_id): model_id = str(model_id) spec = self.specs(model_id) spec['policy'] = loading_policy(spec.get('policy')) entry = self.entries.get(model_id) if entry and entry['spec']['fingerprint'] != spec['fingerprint']: if entry['leases'] or entry['state'] == 'LOADING': raise ValueError('模型配置已变更,请等待当前请求结束') self._discard(entry) entry = None if entry is None: entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None, 'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {}, 'last_used': self.clock(), 'loaded_at': None, 'started_at': None, 'load_order': None} self.entries[model_id] = entry previous = entry['spec']['policy'] entry['spec'] = spec if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']: entry['generation'] += 1 self._put(entry) return entry def _put(self, entry): self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation'])) def start(self): with self.lock: if self.worker is None: self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True) self.worker.start() def submit(self, model_id, start=True): with self.lock: entry = self._entry(model_id) if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']): entry.update(state='UNLOADED', value=None, info={}) if entry['state'] == 'READY': return entry if entry['state'] not in ('LOADING', 'QUEUED'): entry.update(state='QUEUED', error='', future=Future()) entry['generation'] += 1 self._put(entry) if start: self.start() return entry def preload(self, model_ids): # Enqueue the complete startup batch before choosing the highest-priority model. with self.lock: for model_id in model_ids: self.submit(model_id, start=False) self.start() def _work(self): while True: _, order, model_id, generation = self.queue.get() try: with self.lock: entry = self.entries.get(model_id) if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED': continue entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence)) spec = entry['spec'] future = entry['future'] value, info = self.adapter.load(spec) with self.lock: entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock()) future.set_result(True) del value # The loader thread must not keep the last model alive after eviction. except Exception as exc: with self.lock: entry.update(state='FAILED', error=self.adapter.error(exc), value=None) if not entry['future'].done(): entry['future'].set_exception(ValueError(entry['error'])) finally: self.queue.task_done() def acquire(self, model_id, remote=False, timeout=600): with self.lock: entry = self.submit(model_id) token = uuid.uuid4().hex entry['leases'][token] = self.clock() + 3600 if remote else float('inf') future = entry['future'] if entry['state'] != 'READY' else None try: if future: future.result(timeout=timeout) return token except Exception: self.release(model_id, token) raise def release(self, model_id, token): with self.lock: entry = self.entries.get(str(model_id)) if entry: entry['leases'].pop(token, None) entry['last_used'] = self.clock() @contextlib.contextmanager def inference(self, model_id): token = self.acquire(model_id) try: with self.lock: value = self.entries[str(model_id)]['value'] yield value finally: self.release(model_id, token) def _discard(self, entry): if entry['value'] is not None: value = entry['value'] self.adapter.unload(entry['spec'], value) entry['value'] = None del value self.adapter.collect() entry.update(state='UNLOADED', loaded_at=None, info={}) def unload(self, model_id): with self.lock: entry = self._entry(model_id) self._expire_leases(entry) if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'): raise ValueError('模型正在使用或加载,请稍后再试') if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']): entry['value'] = True self._discard(entry) def configure(self, model_id): with self.lock: entry = self._entry(model_id) warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' if warm_again: entry.update(state='UNLOADED', value=None) if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD': self.submit(model_id) def _expire_leases(self, entry): now = self.clock() entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now} def sweep(self): with self.lock: for entry in list(self.entries.values()): self._expire_leases(entry) policy = entry['spec']['policy'] # Ollama owns its idle timer; do not unload models used by other clients. if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and self.clock() - entry['last_used'] >= policy['idle_seconds']): self._discard(entry) def status(self, model_id): with self.lock: entry = self._entry(model_id) self._expire_leases(entry) # Reconcile native residency, including expiration outside this process. if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'): info = self.adapter.probe(entry['spec']) if info: entry.update(state='READY', info=info, value=True) elif entry['state'] == 'READY': entry.update(state='UNLOADED', info={}, value=None, loaded_at=None) return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'], 'active_requests': len(entry['leases']), 'error': entry['error'], 'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'], 'load_order': entry['load_order'], **entry['info']}