"""Weight owner for local resources; shared by both assistants through model IDs.""" import gc import hashlib import json import logging import threading import time from urllib.parse import urlsplit, urlunsplit import requests from django.db import close_old_connections from common.model_runtime import ModelRuntime, loading_policy logger = logging.getLogger('maxkb') SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}} def driver_for(provider, model_type): driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider) return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None def model_spec(model_id): from local_model.models import Model from local_model.serializers.rsa_util import rsa_long_decrypt try: model = Model.objects.get(id=model_id, status='SUCCESS') except Model.DoesNotExist: raise ValueError('模型不存在或尚不可用') driver = driver_for(model.provider, model.model_type) if not driver: raise ValueError('该模型由外部服务运行,不支持本地加载策略') fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest() return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver, 'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)), 'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))} class WeightAdapter: @staticmethod def _request(spec, route, data=None): url = urlsplit(spec['credential'].get('api_base', '')) if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password: raise ValueError('模型服务地址无效') path = url.path.rstrip('/') if path.endswith('/v1'): path = path[:-3] endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', '')) key = spec['credential'].get('api_key') headers = {'Authorization': 'Bearer ' + key} if key else {} response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers, timeout=(5, 600 if data is not None else 5), allow_redirects=False) response.raise_for_status() return response.json() def probe(self, spec): try: tags = self._request(spec, '/api/ps').get('models', []) except requests.RequestException as exc: raise ValueError('模型推理服务不可用') from exc name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest' item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None) if item: vram = item.get('size_vram', 0) return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)} def load(self, spec): close_old_connections() try: if spec['driver'] == 'ollama': keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds'] if spec['type'] == 'EMBEDDING': self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive}) else: self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False, 'keep_alive': keep_alive}) info = self.probe(spec) if not info: raise ValueError('模型未驻留') logger.info('Model weights ready: %s', spec['id']) return True, info from local_model.serializers.model_apply_serializers import get_local_model instance = get_local_model(spec['model'], use_local=True) if spec['type'] == 'EMBEDDING': instance.embed_query('模型加载检查') else: from langchain_core.documents import Document instance.compress_documents([Document(page_content='模型加载检查')], '模型') client = getattr(instance, '_client', None) or getattr(instance, 'client', None) device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu')) info = {'device': device} try: import torch if device.startswith('cuda'): info['process_gpu_bytes'] = torch.cuda.memory_allocated(device) except ImportError: pass logger.info('Model weights ready: %s', spec['id']) return instance, info finally: close_old_connections() def unload(self, spec, value): if spec['driver'] == 'ollama': # Refuse to unload a native model shared by another configured resource. from local_model.models import Model if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists(): raise ValueError('该模型被其他模型资源共用,不能单独卸载') if spec['type'] == 'EMBEDDING': self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0}) else: self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False}) for _ in range(50): if self.probe(spec) is None: return time.sleep(.2) raise ValueError('模型卸载未完成,请稍后重试') @staticmethod def collect(): gc.collect() try: import torch if torch.cuda.is_initialized(): torch.cuda.empty_cache() except ImportError: pass @staticmethod def error(exc): name = type(exc).__name__ logger.warning('Model weight load failed: %s', name) if 'OutOfMemory' in name: return '模型加载失败:显存或内存不足' return '模型加载失败,请检查模型文件、运行设备及服务状态' manager = ModelRuntime(model_spec, WeightAdapter()) _startup_lock = threading.Lock() _started = False def start_runtime(): global _started with _startup_lock: if _started: return _started = True def run(): from django.apps import apps while not apps.ready: time.sleep(.05) from local_model.models import Model try: models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD') ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)] manager.preload(ids) logger.info('Startup model preloads queued: %s', len(ids)) except Exception as exc: logger.warning('Startup preload discovery failed: %s', type(exc).__name__) finally: close_old_connections() while True: time.sleep(10) try: manager.sweep() except Exception as exc: logger.warning('Model idle sweep failed: %s', type(exc).__name__) finally: close_old_connections() threading.Thread(target=run, name='model-startup', daemon=True).start()