173 lines
7.5 KiB
Python
173 lines
7.5 KiB
Python
"""Weight owner for local resources; shared by both assistants through model IDs."""
|
|
import gc
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
import requests
|
|
from django.db import close_old_connections
|
|
|
|
from common.model_runtime import ModelRuntime, loading_policy
|
|
|
|
logger = logging.getLogger('maxkb')
|
|
SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}}
|
|
|
|
|
|
def driver_for(provider, model_type):
|
|
driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider)
|
|
return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None
|
|
|
|
|
|
def model_spec(model_id):
|
|
from local_model.models import Model
|
|
from local_model.serializers.rsa_util import rsa_long_decrypt
|
|
try:
|
|
model = Model.objects.get(id=model_id, status='SUCCESS')
|
|
except Model.DoesNotExist:
|
|
raise ValueError('模型不存在或尚不可用')
|
|
driver = driver_for(model.provider, model.model_type)
|
|
if not driver:
|
|
raise ValueError('该模型由外部服务运行,不支持本地加载策略')
|
|
fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest()
|
|
return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver,
|
|
'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)),
|
|
'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))}
|
|
|
|
|
|
class WeightAdapter:
|
|
@staticmethod
|
|
def _request(spec, route, data=None):
|
|
url = urlsplit(spec['credential'].get('api_base', ''))
|
|
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password:
|
|
raise ValueError('模型服务地址无效')
|
|
path = url.path.rstrip('/')
|
|
if path.endswith('/v1'):
|
|
path = path[:-3]
|
|
endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', ''))
|
|
key = spec['credential'].get('api_key')
|
|
headers = {'Authorization': 'Bearer ' + key} if key else {}
|
|
response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers,
|
|
timeout=(5, 600 if data is not None else 5), allow_redirects=False)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
def probe(self, spec):
|
|
try:
|
|
tags = self._request(spec, '/api/ps').get('models', [])
|
|
except requests.RequestException as exc:
|
|
raise ValueError('模型推理服务不可用') from exc
|
|
name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest'
|
|
item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None)
|
|
if item:
|
|
vram = item.get('size_vram', 0)
|
|
return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)}
|
|
|
|
def load(self, spec):
|
|
close_old_connections()
|
|
try:
|
|
if spec['driver'] == 'ollama':
|
|
keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds']
|
|
if spec['type'] == 'EMBEDDING':
|
|
self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive})
|
|
else:
|
|
self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False,
|
|
'keep_alive': keep_alive})
|
|
info = self.probe(spec)
|
|
if not info:
|
|
raise ValueError('模型未驻留')
|
|
logger.info('Model weights ready: %s', spec['id'])
|
|
return True, info
|
|
from local_model.serializers.model_apply_serializers import get_local_model
|
|
instance = get_local_model(spec['model'], use_local=True)
|
|
if spec['type'] == 'EMBEDDING':
|
|
instance.embed_query('模型加载检查')
|
|
else:
|
|
from langchain_core.documents import Document
|
|
instance.compress_documents([Document(page_content='模型加载检查')], '模型')
|
|
client = getattr(instance, '_client', None) or getattr(instance, 'client', None)
|
|
device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu'))
|
|
info = {'device': device}
|
|
try:
|
|
import torch
|
|
if device.startswith('cuda'):
|
|
info['process_gpu_bytes'] = torch.cuda.memory_allocated(device)
|
|
except ImportError:
|
|
pass
|
|
logger.info('Model weights ready: %s', spec['id'])
|
|
return instance, info
|
|
finally:
|
|
close_old_connections()
|
|
|
|
def unload(self, spec, value):
|
|
if spec['driver'] == 'ollama':
|
|
# Refuse to unload a native model shared by another configured resource.
|
|
from local_model.models import Model
|
|
if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists():
|
|
raise ValueError('该模型被其他模型资源共用,不能单独卸载')
|
|
if spec['type'] == 'EMBEDDING':
|
|
self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0})
|
|
else:
|
|
self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False})
|
|
for _ in range(50):
|
|
if self.probe(spec) is None:
|
|
return
|
|
time.sleep(.2)
|
|
raise ValueError('模型卸载未完成,请稍后重试')
|
|
|
|
@staticmethod
|
|
def collect():
|
|
gc.collect()
|
|
try:
|
|
import torch
|
|
if torch.cuda.is_initialized():
|
|
torch.cuda.empty_cache()
|
|
except ImportError:
|
|
pass
|
|
|
|
@staticmethod
|
|
def error(exc):
|
|
name = type(exc).__name__
|
|
logger.warning('Model weight load failed: %s', name)
|
|
if 'OutOfMemory' in name:
|
|
return '模型加载失败:显存或内存不足'
|
|
return '模型加载失败,请检查模型文件、运行设备及服务状态'
|
|
|
|
|
|
manager = ModelRuntime(model_spec, WeightAdapter())
|
|
_startup_lock = threading.Lock()
|
|
_started = False
|
|
|
|
|
|
def start_runtime():
|
|
global _started
|
|
with _startup_lock:
|
|
if _started:
|
|
return
|
|
_started = True
|
|
def run():
|
|
from django.apps import apps
|
|
while not apps.ready:
|
|
time.sleep(.05)
|
|
from local_model.models import Model
|
|
try:
|
|
models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD')
|
|
ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)]
|
|
manager.preload(ids)
|
|
logger.info('Startup model preloads queued: %s', len(ids))
|
|
except Exception as exc:
|
|
logger.warning('Startup preload discovery failed: %s', type(exc).__name__)
|
|
finally:
|
|
close_old_connections()
|
|
while True:
|
|
time.sleep(10)
|
|
try:
|
|
manager.sweep()
|
|
except Exception as exc:
|
|
logger.warning('Model idle sweep failed: %s', type(exc).__name__)
|
|
finally:
|
|
close_old_connections()
|
|
threading.Thread(target=run, name='model-startup', daemon=True).start()
|