58 lines
2.0 KiB
Python
58 lines
2.0 KiB
Python
# coding=utf-8
|
||
"""
|
||
@project: maxkb
|
||
@Author:虎
|
||
@file: tokenizer_manage_config.py
|
||
@date:2024/4/28 10:17
|
||
@desc:
|
||
"""
|
||
|
||
import os
|
||
import logging
|
||
import threading
|
||
from pathlib import Path
|
||
|
||
class MKTokenizer:
|
||
def __init__(self, tokenizer):
|
||
self.tokenizer = tokenizer
|
||
|
||
def encode(self, text):
|
||
return self.tokenizer.encode(text).ids
|
||
|
||
|
||
class ByteTokenEstimate:
|
||
"""Conservative token budget when an offline tokenizer is unavailable.
|
||
|
||
Provider-reported usage remains authoritative. This only supplies local
|
||
estimates for prompt budgeting; it never downloads model files.
|
||
"""
|
||
def encode(self, text):
|
||
return list(text.encode('utf-8'))
|
||
|
||
|
||
class TokenizerManage:
|
||
tokenizer = None
|
||
_lock = threading.Lock()
|
||
|
||
@staticmethod
|
||
def get_tokenizer():
|
||
if TokenizerManage.tokenizer is not None:
|
||
return TokenizerManage.tokenizer
|
||
with TokenizerManage._lock:
|
||
if TokenizerManage.tokenizer is not None:
|
||
return TokenizerManage.tokenizer
|
||
from tokenizers import Tokenizer
|
||
model_path = Path(os.environ.get('MAXKB_TOKENIZER_PATH') or
|
||
'/opt/maxkb-app/model/tokenizer/models--bert-base-cased')
|
||
try:
|
||
path = model_path if model_path.is_file() else model_path / 'tokenizer.json'
|
||
if not path.is_file():
|
||
snapshot = (model_path / 'refs/main').read_text(encoding='utf-8').strip()
|
||
path = model_path / 'snapshots' / snapshot / 'tokenizer.json'
|
||
TokenizerManage.tokenizer = MKTokenizer(Tokenizer.from_file(str(path)))
|
||
except Exception:
|
||
# A missing/corrupt tokenizer must not prevent remote-model chat.
|
||
logging.getLogger(__name__).warning('本地分词器不可用,使用保守字节估算;实际用量以模型返回为准')
|
||
TokenizerManage.tokenizer = ByteTokenEstimate()
|
||
return TokenizerManage.tokenizer
|