66 lines
3.0 KiB
Python
66 lines
3.0 KiB
Python
"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
|
||
import math
|
||
import unicodedata
|
||
|
||
from langchain_core.embeddings import Embeddings
|
||
|
||
|
||
def validate_params(params):
|
||
if (not isinstance(params, dict) or len(params) > 52 or
|
||
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
|
||
raise ValueError('可选参数最多50项')
|
||
result, names = {}, set()
|
||
for key, value in params.items():
|
||
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
|
||
raise ValueError('参数名称须为1~64个字符')
|
||
key = key.strip()
|
||
normalized = unicodedata.normalize('NFKC', key).casefold()
|
||
if normalized in names:
|
||
raise ValueError('参数名称不能重复')
|
||
names.add(normalized)
|
||
if value is not None and (not isinstance(value, (str, int, float, bool)) or
|
||
isinstance(value, str) and len(value) > 2000 or
|
||
isinstance(value, float) and not math.isfinite(value)):
|
||
raise ValueError('参数值须为2000字以内的文本或数值')
|
||
result[key] = value
|
||
batch = result.get('batch_size')
|
||
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
|
||
raise ValueError('批处理数量须为1~256的整数')
|
||
if result.get('normalize') is not None and type(result['normalize']) is not bool:
|
||
raise ValueError('向量归一化参数无效')
|
||
for key in ('query_prefix', 'document_prefix'):
|
||
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
|
||
raise ValueError('向量指令须为2000字以内的文本')
|
||
return {key: value for key, value in result.items() if value is not None and
|
||
(value != '' or key not in {'query_prefix', 'document_prefix'})}
|
||
|
||
|
||
def knowledge_params(knowledge):
|
||
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
|
||
return {key: value for key, value in params.items()
|
||
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
|
||
|
||
|
||
class ConfiguredEmbedding(Embeddings):
|
||
def __init__(self, model, params):
|
||
self.model = model
|
||
self.params = params
|
||
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
|
||
|
||
def _vector(self, vector):
|
||
if not self.params.get('normalize'):
|
||
return vector
|
||
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
|
||
return [float(v) / norm for v in vector] if norm else vector
|
||
|
||
def embed_query(self, text):
|
||
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
|
||
|
||
def embed_documents(self, texts):
|
||
prefix = self.params.get('document_prefix', '')
|
||
result = []
|
||
for start in range(0, len(texts), self.batch_size):
|
||
result.extend(self._vector(v) for v in self.model.embed_documents(
|
||
[prefix + text for text in texts[start:start + self.batch_size]]))
|
||
return result
|