UnisKB/apps/common/config/tokenizer_manage_config.py

58 lines
2.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

# coding=utf-8
"""
@project: maxkb
@Author:虎
@file: tokenizer_manage_config.py
@date:2024/4/28 10:17
@desc:
"""
import os
import logging
import threading
from pathlib import Path
class MKTokenizer:
def __init__(self, tokenizer):
self.tokenizer = tokenizer
def encode(self, text):
return self.tokenizer.encode(text).ids
class ByteTokenEstimate:
"""Conservative token budget when an offline tokenizer is unavailable.
Provider-reported usage remains authoritative. This only supplies local
estimates for prompt budgeting; it never downloads model files.
"""
def encode(self, text):
return list(text.encode('utf-8'))
class TokenizerManage:
tokenizer = None
_lock = threading.Lock()
@staticmethod
def get_tokenizer():
if TokenizerManage.tokenizer is not None:
return TokenizerManage.tokenizer
with TokenizerManage._lock:
if TokenizerManage.tokenizer is not None:
return TokenizerManage.tokenizer
from tokenizers import Tokenizer
model_path = Path(os.environ.get('MAXKB_TOKENIZER_PATH') or
'/opt/maxkb-app/model/tokenizer/models--bert-base-cased')
try:
path = model_path if model_path.is_file() else model_path / 'tokenizer.json'
if not path.is_file():
snapshot = (model_path / 'refs/main').read_text(encoding='utf-8').strip()
path = model_path / 'snapshots' / snapshot / 'tokenizer.json'
TokenizerManage.tokenizer = MKTokenizer(Tokenizer.from_file(str(path)))
except Exception:
# A missing/corrupt tokenizer must not prevent remote-model chat.
logging.getLogger(__name__).warning('本地分词器不可用,使用保守字节估算;实际用量以模型返回为准')
TokenizerManage.tokenizer = ByteTokenEstimate()
return TokenizerManage.tokenizer