# coding=utf-8 """ @project: maxkb @Author:虎 @file: tokenizer_manage_config.py @date:2024/4/28 10:17 @desc: """ import os import logging import threading from pathlib import Path class MKTokenizer: def __init__(self, tokenizer): self.tokenizer = tokenizer def encode(self, text): return self.tokenizer.encode(text).ids class ByteTokenEstimate: """Conservative token budget when an offline tokenizer is unavailable. Provider-reported usage remains authoritative. This only supplies local estimates for prompt budgeting; it never downloads model files. """ def encode(self, text): return list(text.encode('utf-8')) class TokenizerManage: tokenizer = None _lock = threading.Lock() @staticmethod def get_tokenizer(): if TokenizerManage.tokenizer is not None: return TokenizerManage.tokenizer with TokenizerManage._lock: if TokenizerManage.tokenizer is not None: return TokenizerManage.tokenizer from tokenizers import Tokenizer model_path = Path(os.environ.get('MAXKB_TOKENIZER_PATH') or '/opt/maxkb-app/model/tokenizer/models--bert-base-cased') try: path = model_path if model_path.is_file() else model_path / 'tokenizer.json' if not path.is_file(): snapshot = (model_path / 'refs/main').read_text(encoding='utf-8').strip() path = model_path / 'snapshots' / snapshot / 'tokenizer.json' TokenizerManage.tokenizer = MKTokenizer(Tokenizer.from_file(str(path))) except Exception: # A missing/corrupt tokenizer must not prevent remote-model chat. logging.getLogger(__name__).warning('本地分词器不可用,使用保守字节估算;实际用量以模型返回为准') TokenizerManage.tokenizer = ByteTokenEstimate() return TokenizerManage.tokenizer