UnisKB/apps/knowledge/serializers/knowledge_ext.py

216 lines
10 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

# coding=utf-8
"""知识库扩展配置序列化器:业务元数据、分段规则。"""
from collections.abc import Mapping
import re
from django.db import transaction
from django.db.models import QuerySet
from django.utils import timezone
from rest_framework import serializers
from common.exception.app_exception import AppApiException
from knowledge.models import Knowledge
METADATA_FIELDS = [
{"field": "business_domain", "label": "业务领域"},
{"field": "data_source", "label": "数据来源"},
{"field": "owner_dept", "label": "责任部门"},
{"field": "business_version", "label": "业务版本"},
]
METADATA_MAX_LEN = 64
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge:
query = QuerySet(Knowledge)
if lock:
query = query.select_for_update()
knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first()
if knowledge is None:
raise AppApiException(500, "知识库不存在")
return knowledge
def _creator_candidates(workspace_id):
from users.models import User, UserActivationApplication
from common.database_model_manage.database_model_manage import DatabaseModelManage
from django.db.models import Q
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
query = User.objects.filter(is_active=True).exclude(id__in=pending)
membership = DatabaseModelManage.get_model('workspace_user_role_mapping')
if membership is not None:
ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True)
query = query.filter(Q(id__in=ids) | Q(role='ADMIN'))
return query
def _metadata_fields(business):
# An explicit empty list means all custom fields were removed; do not revive old fields.
if isinstance(business.get('custom_fields'), list):
return business['custom_fields']
return [{'name': item['label'], 'value': business[item['field']]}
for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')]
def _validate_custom_fields(items):
import unicodedata
if not isinstance(items, list) or len(items) > 50:
raise AppApiException(500, '元数据最多50项')
reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'}
result, names = [], set()
for item in items:
if not isinstance(item, Mapping) or set(item) != {'name', 'value'}:
raise AppApiException(500, '元数据需包含名称和值')
name, value = item['name'], item['value']
if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name):
raise AppApiException(500, '元数据名称须为1~64个字符')
name = name.strip()
normalized = unicodedata.normalize('NFKC', name).casefold()
if normalized in reserved:
raise AppApiException(500, '不能使用固定属性名称')
if normalized in names:
raise AppApiException(500, '元数据名称不能重复')
if not isinstance(value, str) or len(value) > 2000:
raise AppApiException(500, '元数据值须为2000字以内的文本')
names.add(normalized)
result.append({'name': name, 'value': value.strip()})
return result
class KnowledgeMetadataSerializer(serializers.Serializer):
"""Fixed system attributes plus freely named business metadata; save atomically."""
@staticmethod
def one(workspace_id: str, knowledge_id: str):
from users.models import User
knowledge = _get_knowledge(workspace_id, knowledge_id)
data = (knowledge.meta or {}).get('business_metadata') or {}
creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None
candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False}
for u in _creator_candidates(workspace_id).order_by('nick_name')]
if creator and not any(u['id'] == str(creator.id) for u in candidates):
candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True})
return {
'knowledge_id': str(knowledge.id),
'user_id': str(knowledge.user_id) if knowledge.user_id else '',
'creator_name': (creator.nick_name or creator.username) if creator else '',
'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time),
'creator_options': candidates,
'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)],
'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '',
}
@staticmethod
@transaction.atomic
def update(workspace_id: str, knowledge_id: str, user, data: dict):
if not isinstance(data, Mapping):
raise AppApiException(500, '元数据必须为属性对象')
legacy = {item['field']: item['label'] for item in METADATA_FIELDS}
if set(data) - (set(legacy) | {'fields', 'user_id'}):
raise AppApiException(500, '包含不支持的元数据属性')
if 'fields' in data and set(data) & set(legacy):
raise AppApiException(500, '不能混用新旧元数据格式')
fields = _validate_custom_fields(data['fields']) if 'fields' in data else None
values = {}
for key, label in legacy.items():
if key not in data:
continue
value = '' if data[key] is None else data[key]
if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN:
raise AppApiException(500, f'{label}须为64字以内的文本')
values[key] = value.strip()
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id):
try:
creator_id = serializers.UUIDField().run_validation(data['user_id'])
except serializers.ValidationError:
raise AppApiException(500, '请选择有效创建者')
if not _creator_candidates(workspace_id).filter(id=creator_id).exists():
raise AppApiException(500, '创建者须为已激活且启用的成员')
knowledge.user_id = creator_id
meta = dict(knowledge.meta or {})
business = dict(meta.get('business_metadata') or {})
if fields is not None:
business['custom_fields'] = fields
elif 'custom_fields' in business:
current = list(_metadata_fields(business))
for key, value in values.items():
existing = next((item for item in current if item['name'] == legacy[key]), None)
if existing is not None:
existing['value'] = value
elif value:
current.append({'name': legacy[key], 'value': value})
business['custom_fields'] = _validate_custom_fields(current)
business.update(values)
business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username,
updated_at=_fmt(timezone.now()))
meta['business_metadata'] = business
knowledge.meta = meta
knowledge.save(update_fields=['meta', 'user', 'update_time'])
return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)
class KnowledgeSegmentationSerializer(serializers.Serializer):
"""知识库分段规则:查看 / 修改。
chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度;
保存后新建文档上传时默认使用该规则,上传时可覆盖。
"""
DEFAULTS = {"chunk_size": 500, "chunk_overlap": 0}
@staticmethod
def one(workspace_id: str, knowledge_id: str):
knowledge = _get_knowledge(workspace_id, knowledge_id)
data = dict((knowledge.meta or {}).get('segmentation') or {})
result = {
"chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"],
"chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None
else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"],
}
result.update({
"updated_by": data.get("updated_by_name") or "",
"updated_at": data.get("updated_at") or "",
"knowledge_id": str(knowledge.id),
})
return result
@staticmethod
@transaction.atomic
def update(workspace_id: str, knowledge_id: str, user, data: dict):
if not isinstance(data, Mapping):
raise AppApiException(500, "分段规则必须为属性对象")
def integer(value):
if isinstance(value, bool) or not isinstance(value, (int, str)):
raise AppApiException(500, "分段长度/重叠长度必须为整数")
if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value):
raise AppApiException(500, "分段长度/重叠长度必须为整数")
return int(value)
chunk_size = integer(data.get("chunk_size"))
chunk_overlap = integer(data.get("chunk_overlap", 0))
if chunk_size < 50 or chunk_size > 100000:
raise AppApiException(500, "分段长度取值范围为 50~100000")
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
meta = dict(knowledge.meta or {})
meta['segmentation'] = {
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap,
"updated_by": str(user.id),
"updated_by_name": getattr(user, 'nick_name', '') or user.username,
"updated_at": _fmt(timezone.now()),
}
knowledge.meta = meta
knowledge.save(update_fields=["meta", "update_time"])
return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)