216 lines
10 KiB
Python
216 lines
10 KiB
Python
# coding=utf-8
|
||
"""知识库扩展配置序列化器:业务元数据、分段规则。"""
|
||
from collections.abc import Mapping
|
||
import re
|
||
|
||
from django.db import transaction
|
||
from django.db.models import QuerySet
|
||
from django.utils import timezone
|
||
from rest_framework import serializers
|
||
|
||
from common.exception.app_exception import AppApiException
|
||
from knowledge.models import Knowledge
|
||
|
||
METADATA_FIELDS = [
|
||
{"field": "business_domain", "label": "业务领域"},
|
||
{"field": "data_source", "label": "数据来源"},
|
||
{"field": "owner_dept", "label": "责任部门"},
|
||
{"field": "business_version", "label": "业务版本"},
|
||
]
|
||
METADATA_MAX_LEN = 64
|
||
|
||
|
||
def _fmt(value):
|
||
if not value:
|
||
return ""
|
||
try:
|
||
value = timezone.localtime(value)
|
||
except Exception:
|
||
pass
|
||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge:
|
||
query = QuerySet(Knowledge)
|
||
if lock:
|
||
query = query.select_for_update()
|
||
knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first()
|
||
if knowledge is None:
|
||
raise AppApiException(500, "知识库不存在")
|
||
return knowledge
|
||
|
||
|
||
def _creator_candidates(workspace_id):
|
||
from users.models import User, UserActivationApplication
|
||
from common.database_model_manage.database_model_manage import DatabaseModelManage
|
||
from django.db.models import Q
|
||
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
|
||
query = User.objects.filter(is_active=True).exclude(id__in=pending)
|
||
membership = DatabaseModelManage.get_model('workspace_user_role_mapping')
|
||
if membership is not None:
|
||
ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True)
|
||
query = query.filter(Q(id__in=ids) | Q(role='ADMIN'))
|
||
return query
|
||
|
||
|
||
def _metadata_fields(business):
|
||
# An explicit empty list means all custom fields were removed; do not revive old fields.
|
||
if isinstance(business.get('custom_fields'), list):
|
||
return business['custom_fields']
|
||
return [{'name': item['label'], 'value': business[item['field']]}
|
||
for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')]
|
||
|
||
|
||
def _validate_custom_fields(items):
|
||
import unicodedata
|
||
if not isinstance(items, list) or len(items) > 50:
|
||
raise AppApiException(500, '元数据最多50项')
|
||
reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'}
|
||
result, names = [], set()
|
||
for item in items:
|
||
if not isinstance(item, Mapping) or set(item) != {'name', 'value'}:
|
||
raise AppApiException(500, '元数据需包含名称和值')
|
||
name, value = item['name'], item['value']
|
||
if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name):
|
||
raise AppApiException(500, '元数据名称须为1~64个字符')
|
||
name = name.strip()
|
||
normalized = unicodedata.normalize('NFKC', name).casefold()
|
||
if normalized in reserved:
|
||
raise AppApiException(500, '不能使用固定属性名称')
|
||
if normalized in names:
|
||
raise AppApiException(500, '元数据名称不能重复')
|
||
if not isinstance(value, str) or len(value) > 2000:
|
||
raise AppApiException(500, '元数据值须为2000字以内的文本')
|
||
names.add(normalized)
|
||
result.append({'name': name, 'value': value.strip()})
|
||
return result
|
||
|
||
|
||
class KnowledgeMetadataSerializer(serializers.Serializer):
|
||
"""Fixed system attributes plus freely named business metadata; save atomically."""
|
||
|
||
@staticmethod
|
||
def one(workspace_id: str, knowledge_id: str):
|
||
from users.models import User
|
||
knowledge = _get_knowledge(workspace_id, knowledge_id)
|
||
data = (knowledge.meta or {}).get('business_metadata') or {}
|
||
creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None
|
||
candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False}
|
||
for u in _creator_candidates(workspace_id).order_by('nick_name')]
|
||
if creator and not any(u['id'] == str(creator.id) for u in candidates):
|
||
candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True})
|
||
return {
|
||
'knowledge_id': str(knowledge.id),
|
||
'user_id': str(knowledge.user_id) if knowledge.user_id else '',
|
||
'creator_name': (creator.nick_name or creator.username) if creator else '',
|
||
'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time),
|
||
'creator_options': candidates,
|
||
'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)],
|
||
'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '',
|
||
}
|
||
|
||
@staticmethod
|
||
@transaction.atomic
|
||
def update(workspace_id: str, knowledge_id: str, user, data: dict):
|
||
if not isinstance(data, Mapping):
|
||
raise AppApiException(500, '元数据必须为属性对象')
|
||
legacy = {item['field']: item['label'] for item in METADATA_FIELDS}
|
||
if set(data) - (set(legacy) | {'fields', 'user_id'}):
|
||
raise AppApiException(500, '包含不支持的元数据属性')
|
||
if 'fields' in data and set(data) & set(legacy):
|
||
raise AppApiException(500, '不能混用新旧元数据格式')
|
||
fields = _validate_custom_fields(data['fields']) if 'fields' in data else None
|
||
values = {}
|
||
for key, label in legacy.items():
|
||
if key not in data:
|
||
continue
|
||
value = '' if data[key] is None else data[key]
|
||
if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN:
|
||
raise AppApiException(500, f'{label}须为64字以内的文本')
|
||
values[key] = value.strip()
|
||
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
|
||
if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id):
|
||
try:
|
||
creator_id = serializers.UUIDField().run_validation(data['user_id'])
|
||
except serializers.ValidationError:
|
||
raise AppApiException(500, '请选择有效创建者')
|
||
if not _creator_candidates(workspace_id).filter(id=creator_id).exists():
|
||
raise AppApiException(500, '创建者须为已激活且启用的成员')
|
||
knowledge.user_id = creator_id
|
||
meta = dict(knowledge.meta or {})
|
||
business = dict(meta.get('business_metadata') or {})
|
||
if fields is not None:
|
||
business['custom_fields'] = fields
|
||
elif 'custom_fields' in business:
|
||
current = list(_metadata_fields(business))
|
||
for key, value in values.items():
|
||
existing = next((item for item in current if item['name'] == legacy[key]), None)
|
||
if existing is not None:
|
||
existing['value'] = value
|
||
elif value:
|
||
current.append({'name': legacy[key], 'value': value})
|
||
business['custom_fields'] = _validate_custom_fields(current)
|
||
business.update(values)
|
||
business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username,
|
||
updated_at=_fmt(timezone.now()))
|
||
meta['business_metadata'] = business
|
||
knowledge.meta = meta
|
||
knowledge.save(update_fields=['meta', 'user', 'update_time'])
|
||
return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)
|
||
|
||
|
||
class KnowledgeSegmentationSerializer(serializers.Serializer):
|
||
"""知识库分段规则:查看 / 修改。
|
||
|
||
chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度;
|
||
保存后新建文档上传时默认使用该规则,上传时可覆盖。
|
||
"""
|
||
|
||
DEFAULTS = {"chunk_size": 500, "chunk_overlap": 0}
|
||
|
||
@staticmethod
|
||
def one(workspace_id: str, knowledge_id: str):
|
||
knowledge = _get_knowledge(workspace_id, knowledge_id)
|
||
data = dict((knowledge.meta or {}).get('segmentation') or {})
|
||
result = {
|
||
"chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"],
|
||
"chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None
|
||
else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"],
|
||
}
|
||
result.update({
|
||
"updated_by": data.get("updated_by_name") or "",
|
||
"updated_at": data.get("updated_at") or "",
|
||
"knowledge_id": str(knowledge.id),
|
||
})
|
||
return result
|
||
|
||
@staticmethod
|
||
@transaction.atomic
|
||
def update(workspace_id: str, knowledge_id: str, user, data: dict):
|
||
if not isinstance(data, Mapping):
|
||
raise AppApiException(500, "分段规则必须为属性对象")
|
||
def integer(value):
|
||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||
raise AppApiException(500, "分段长度/重叠长度必须为整数")
|
||
if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value):
|
||
raise AppApiException(500, "分段长度/重叠长度必须为整数")
|
||
return int(value)
|
||
chunk_size = integer(data.get("chunk_size"))
|
||
chunk_overlap = integer(data.get("chunk_overlap", 0))
|
||
if chunk_size < 50 or chunk_size > 100000:
|
||
raise AppApiException(500, "分段长度取值范围为 50~100000")
|
||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
|
||
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
|
||
meta = dict(knowledge.meta or {})
|
||
meta['segmentation'] = {
|
||
"chunk_size": chunk_size,
|
||
"chunk_overlap": chunk_overlap,
|
||
"updated_by": str(user.id),
|
||
"updated_by_name": getattr(user, 'nick_name', '') or user.username,
|
||
"updated_at": _fmt(timezone.now()),
|
||
}
|
||
knowledge.meta = meta
|
||
knowledge.save(update_fields=["meta", "update_time"])
|
||
return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)
|