UnisKB/apps/local_model/runtime.py

173 lines
7.5 KiB
Python

"""Weight owner for local resources; shared by both assistants through model IDs."""
import gc
import hashlib
import json
import logging
import threading
import time
from urllib.parse import urlsplit, urlunsplit
import requests
from django.db import close_old_connections
from common.model_runtime import ModelRuntime, loading_policy
logger = logging.getLogger('maxkb')
SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}}
def driver_for(provider, model_type):
driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider)
return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None
def model_spec(model_id):
from local_model.models import Model
from local_model.serializers.rsa_util import rsa_long_decrypt
try:
model = Model.objects.get(id=model_id, status='SUCCESS')
except Model.DoesNotExist:
raise ValueError('模型不存在或尚不可用')
driver = driver_for(model.provider, model.model_type)
if not driver:
raise ValueError('该模型由外部服务运行,不支持本地加载策略')
fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest()
return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver,
'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)),
'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))}
class WeightAdapter:
@staticmethod
def _request(spec, route, data=None):
url = urlsplit(spec['credential'].get('api_base', ''))
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password:
raise ValueError('模型服务地址无效')
path = url.path.rstrip('/')
if path.endswith('/v1'):
path = path[:-3]
endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', ''))
key = spec['credential'].get('api_key')
headers = {'Authorization': 'Bearer ' + key} if key else {}
response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers,
timeout=(5, 600 if data is not None else 5), allow_redirects=False)
response.raise_for_status()
return response.json()
def probe(self, spec):
try:
tags = self._request(spec, '/api/ps').get('models', [])
except requests.RequestException as exc:
raise ValueError('模型推理服务不可用') from exc
name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest'
item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None)
if item:
vram = item.get('size_vram', 0)
return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)}
def load(self, spec):
close_old_connections()
try:
if spec['driver'] == 'ollama':
keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds']
if spec['type'] == 'EMBEDDING':
self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive})
else:
self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False,
'keep_alive': keep_alive})
info = self.probe(spec)
if not info:
raise ValueError('模型未驻留')
logger.info('Model weights ready: %s', spec['id'])
return True, info
from local_model.serializers.model_apply_serializers import get_local_model
instance = get_local_model(spec['model'], use_local=True)
if spec['type'] == 'EMBEDDING':
instance.embed_query('模型加载检查')
else:
from langchain_core.documents import Document
instance.compress_documents([Document(page_content='模型加载检查')], '模型')
client = getattr(instance, '_client', None) or getattr(instance, 'client', None)
device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu'))
info = {'device': device}
try:
import torch
if device.startswith('cuda'):
info['process_gpu_bytes'] = torch.cuda.memory_allocated(device)
except ImportError:
pass
logger.info('Model weights ready: %s', spec['id'])
return instance, info
finally:
close_old_connections()
def unload(self, spec, value):
if spec['driver'] == 'ollama':
# Refuse to unload a native model shared by another configured resource.
from local_model.models import Model
if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists():
raise ValueError('该模型被其他模型资源共用,不能单独卸载')
if spec['type'] == 'EMBEDDING':
self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0})
else:
self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False})
for _ in range(50):
if self.probe(spec) is None:
return
time.sleep(.2)
raise ValueError('模型卸载未完成,请稍后重试')
@staticmethod
def collect():
gc.collect()
try:
import torch
if torch.cuda.is_initialized():
torch.cuda.empty_cache()
except ImportError:
pass
@staticmethod
def error(exc):
name = type(exc).__name__
logger.warning('Model weight load failed: %s', name)
if 'OutOfMemory' in name:
return '模型加载失败:显存或内存不足'
return '模型加载失败,请检查模型文件、运行设备及服务状态'
manager = ModelRuntime(model_spec, WeightAdapter())
_startup_lock = threading.Lock()
_started = False
def start_runtime():
global _started
with _startup_lock:
if _started:
return
_started = True
def run():
from django.apps import apps
while not apps.ready:
time.sleep(.05)
from local_model.models import Model
try:
models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD')
ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)]
manager.preload(ids)
logger.info('Startup model preloads queued: %s', len(ids))
except Exception as exc:
logger.warning('Startup preload discovery failed: %s', type(exc).__name__)
finally:
close_old_connections()
while True:
time.sleep(10)
try:
manager.sweep()
except Exception as exc:
logger.warning('Model idle sweep failed: %s', type(exc).__name__)
finally:
close_old_connections()
threading.Thread(target=run, name='model-startup', daemon=True).start()