UnisKB/apps/knowledge/embedding_params.py

66 lines
3.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
import math
import unicodedata
from langchain_core.embeddings import Embeddings
def validate_params(params):
if (not isinstance(params, dict) or len(params) > 52 or
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
raise ValueError('可选参数最多50项')
result, names = {}, set()
for key, value in params.items():
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
raise ValueError('参数名称须为1~64个字符')
key = key.strip()
normalized = unicodedata.normalize('NFKC', key).casefold()
if normalized in names:
raise ValueError('参数名称不能重复')
names.add(normalized)
if value is not None and (not isinstance(value, (str, int, float, bool)) or
isinstance(value, str) and len(value) > 2000 or
isinstance(value, float) and not math.isfinite(value)):
raise ValueError('参数值须为2000字以内的文本或数值')
result[key] = value
batch = result.get('batch_size')
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
raise ValueError('批处理数量须为1~256的整数')
if result.get('normalize') is not None and type(result['normalize']) is not bool:
raise ValueError('向量归一化参数无效')
for key in ('query_prefix', 'document_prefix'):
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
raise ValueError('向量指令须为2000字以内的文本')
return {key: value for key, value in result.items() if value is not None and
(value != '' or key not in {'query_prefix', 'document_prefix'})}
def knowledge_params(knowledge):
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
return {key: value for key, value in params.items()
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
class ConfiguredEmbedding(Embeddings):
def __init__(self, model, params):
self.model = model
self.params = params
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
def _vector(self, vector):
if not self.params.get('normalize'):
return vector
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
return [float(v) / norm for v in vector] if norm else vector
def embed_query(self, text):
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
def embed_documents(self, texts):
prefix = self.params.get('document_prefix', '')
result = []
for start in range(0, len(texts), self.batch_size):
result.extend(self._vector(v) for v in self.model.embed_documents(
[prefix + text for text in texts[start:start + self.batch_size]]))
return result