332 lines
18 KiB
Python
332 lines
18 KiB
Python
"""Version snapshots reuse existing retrieval storage without exposing another knowledge base."""
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
from datetime import timedelta
|
||
import uuid_utils.compat as uuid
|
||
from django.db import connection, transaction
|
||
from django.db.models import Q, OuterRef, Subquery
|
||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||
from django.utils import timezone
|
||
from rest_framework.exceptions import ValidationError
|
||
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
|
||
from knowledge.models import (Knowledge, KnowledgeVersion, KnowledgeVersionLease, Document, Paragraph,
|
||
Problem, ProblemParagraphMapping, Tag, DocumentTag, Embedding, File, TaskType, State, Status)
|
||
|
||
SNAPSHOT_KEY = '_version_snapshot'
|
||
|
||
|
||
def root_knowledge(workspace, knowledge_id, lock=False):
|
||
query = Knowledge.objects.filter(pk=knowledge_id, workspace_id=workspace).exclude(meta__has_key=SNAPSHOT_KEY)
|
||
if lock:
|
||
query = query.select_for_update()
|
||
obj = query.first()
|
||
if obj is None:
|
||
raise NotFound404(404, '知识库不存在')
|
||
return obj
|
||
|
||
|
||
def version_data(version):
|
||
return {'id': str(version.id), 'number': version.number, 'name': version.name,
|
||
'description': version.description, 'creator_name': version.creator_name,
|
||
'create_time': version.create_time, 'document_count': version.document_count,
|
||
'paragraph_count': version.paragraph_count, 'vector_count': version.vector_count,
|
||
'search_ready': version.search_ready, 'content_hash': version.content_hash,
|
||
'config': {key: value for key, value in version.config.items()
|
||
if not key.startswith('_') and key != 'model_identity'}}
|
||
|
||
|
||
def remap(value, mapping):
|
||
if isinstance(value, dict):
|
||
return {k: remap(v, mapping) for k, v in value.items()}
|
||
if isinstance(value, list):
|
||
return [remap(v, mapping) for v in value]
|
||
if isinstance(value, str):
|
||
if value in mapping:
|
||
return mapping[value]
|
||
# File links are embedded in Markdown, HTML and document metadata.
|
||
import re
|
||
return re.sub(r'(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
|
||
lambda m: mapping.get(m[0], m[0]), value)
|
||
return value
|
||
|
||
|
||
def copy_rows(model, source_rows, ids, overrides=None):
|
||
objects = []
|
||
for row in source_rows:
|
||
values = {field.attname: copy.deepcopy(getattr(row, field.attname)) for field in model._meta.concrete_fields}
|
||
for field in model._meta.concrete_fields:
|
||
key = field.attname
|
||
value = values[key]
|
||
if key.endswith('_id') or key == 'id':
|
||
values[key] = ids.get(str(value), value)
|
||
elif key in ('content', 'title', 'name', 'desc') and isinstance(value, str):
|
||
import re
|
||
values[key] = re.sub(r'(oss/file/)([0-9a-fA-F-]{36})',
|
||
lambda match: match[1] + str(ids.get(match[2], match[2])), value)
|
||
elif isinstance(value, (dict, list, str)):
|
||
values[key] = remap(value, ids)
|
||
values.update(overrides or {})
|
||
objects.append(model(**values))
|
||
return model.objects.bulk_create(objects, batch_size=500)
|
||
|
||
|
||
def _build_index(snapshot_id):
|
||
if connection.vendor != 'postgresql':
|
||
return
|
||
with connection.cursor() as cursor:
|
||
cursor.execute('SELECT vector_dims(embedding) FROM embedding WHERE knowledge_id=%s LIMIT 1', [str(snapshot_id)])
|
||
row = cursor.fetchone()
|
||
if row:
|
||
# Identifier and predicate are generated UUIDs; dimension comes from pgvector.
|
||
name = 'embedding_hnsw_idx_' + str(snapshot_id)
|
||
cursor.execute('CREATE INDEX "' + name + '" ON embedding USING hnsw '
|
||
'((embedding::vector(' + str(int(row[0])) + ')) vector_cosine_ops) '
|
||
"WHERE knowledge_id='" + str(snapshot_id) + "'::uuid")
|
||
|
||
|
||
def create_version(workspace, knowledge_id, user, data):
|
||
name = str(data.get('name') or '').strip()
|
||
description = str(data.get('description') or '').strip()
|
||
if not name or len(name) > 128 or len(description) > 512:
|
||
raise ValidationError('版本名称须为1~128字,说明最多512字')
|
||
nested = connection.in_atomic_block
|
||
with transaction.atomic():
|
||
if connection.vendor == 'postgresql':
|
||
with connection.cursor() as cursor:
|
||
if nested:
|
||
cursor.execute('SHOW transaction_isolation')
|
||
if cursor.fetchone()[0] not in ('repeatable read', 'serializable'):
|
||
raise ValidationError('版本创建需使用独立的一致性事务')
|
||
else:
|
||
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
|
||
root = root_knowledge(workspace, knowledge_id, lock=True)
|
||
documents = list(Document.objects.filter(knowledge=root).order_by('id'))
|
||
if any(Status(doc.status)[kind] in (State.PENDING, State.STARTED)
|
||
for doc in documents for kind in (TaskType.EMBEDDING, TaskType.GENERATE_PROBLEM, TaskType.SYNC)):
|
||
raise ValidationError('文档正在处理,请完成后创建版本')
|
||
paragraphs = list(Paragraph.objects.filter(knowledge=root).order_by('id'))
|
||
problems = list(Problem.objects.filter(knowledge=root).order_by('id'))
|
||
mappings = list(ProblemParagraphMapping.objects.filter(knowledge=root))
|
||
tags = list(Tag.objects.filter(knowledge=root))
|
||
doc_tags = list(DocumentTag.objects.filter(document__knowledge=root))
|
||
vectors = list(Embedding.objects.filter(knowledge=root))
|
||
files = list(File.objects.filter(Q(source_id=str(root.id)) |
|
||
Q(source_id__in=[str(d.id) for d in documents]) | Q(meta__knowledge_id=str(root.id))))
|
||
number = root.version_sequence + 1
|
||
latest = KnowledgeVersion.objects.filter(knowledge=root).first()
|
||
number = max(number, latest.number + 1 if latest else 1)
|
||
version_id, snapshot_id = uuid.uuid7(), uuid.uuid7()
|
||
ids = {str(root.id): str(snapshot_id)}
|
||
for rows in (documents, paragraphs, problems, mappings, tags, doc_tags, vectors, files):
|
||
ids.update({str(row.pk): str(uuid.uuid7()) for row in rows})
|
||
config = {'embedding_model_id': str(root.embedding_model_id) if root.embedding_model_id else None,
|
||
'embedding_params': (root.meta or {}).get('embedding_params', {}),
|
||
'segmentation': (root.meta or {}).get('segmentation', {}), 'knowledge_name': root.name,
|
||
'knowledge_type': root.type, 'metadata': (root.meta or {}).get('business_metadata', {})}
|
||
model = root.embedding_model
|
||
if model:
|
||
config['model_identity'] = {'provider': model.provider, 'model_name': model.model_name,
|
||
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}
|
||
content = {'documents': [[str(d.id), d.name, d.is_active, d.meta] for d in documents],
|
||
'paragraphs': [[str(p.id), str(p.document_id), p.position, p.title, p.content, p.is_active] for p in paragraphs],
|
||
'problems': [[str(p.id), p.content] for p in problems],
|
||
'mappings': [[str(p.problem_id), str(p.paragraph_id)] for p in mappings], 'config': config}
|
||
digest = hashlib.sha256(json.dumps(content, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest()
|
||
config['_document_ids'] = {str(d.id): ids[str(d.id)] for d in documents}
|
||
metadata = copy.deepcopy(root.meta or {})
|
||
metadata.pop('_version_sequence', None)
|
||
metadata.pop('parser_id', None)
|
||
metadata.update({SNAPSHOT_KEY: str(root.id), '_version_id': str(version_id)})
|
||
snapshot = Knowledge.objects.create(id=snapshot_id, name=(root.name[:125] + ' · v' + str(number))[:150],
|
||
desc=root.desc, workspace_id=root.workspace_id, folder_id=root.folder_id, user_id=root.user_id,
|
||
embedding_model_id=root.embedding_model_id, scope=root.scope, type=0,
|
||
file_size_limit=root.file_size_limit, file_count_limit=root.file_count_limit, meta=metadata)
|
||
copy_rows(Document, documents, ids, {'knowledge_id': snapshot_id, 'type': 0})
|
||
copy_rows(Paragraph, paragraphs, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
|
||
copy_rows(Problem, problems, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
|
||
copy_rows(ProblemParagraphMapping, mappings, ids, {'knowledge_id': snapshot_id})
|
||
copy_rows(Tag, tags, ids, {'knowledge_id': snapshot_id})
|
||
copy_rows(DocumentTag, doc_tags, ids)
|
||
# Preserve binary large-object references; only the last File reference frees the bytes.
|
||
cloned_files = copy_rows(File, files, ids, {'source_id': str(snapshot_id)})
|
||
for file in cloned_files:
|
||
file.meta = {**(file.meta or {}), '_version_id': str(version_id), 'knowledge_id': str(root.id)}
|
||
File.objects.bulk_update(cloned_files, ['meta'], batch_size=500)
|
||
copied_vectors = copy_rows(Embedding, vectors, ids, {'knowledge_id': snapshot_id})
|
||
for old, new in zip(vectors, copied_vectors):
|
||
new.source_id = ids.get(str(old.source_id), old.source_id)
|
||
Embedding.objects.bulk_update(copied_vectors, ['source_id'], batch_size=500)
|
||
active_docs = {d.id for d in documents if d.is_active}
|
||
active_paragraphs = {p.id for p in paragraphs if p.is_active and p.document_id in active_docs}
|
||
indexed = {v.paragraph_id for v in vectors if v.is_active}
|
||
search_ready = bool(model and active_paragraphs and active_paragraphs <= indexed)
|
||
_build_index(snapshot_id)
|
||
version = KnowledgeVersion.objects.create(id=version_id, knowledge=root, snapshot=snapshot,
|
||
number=number, name=name, description=description, creator=user,
|
||
creator_name=user.nick_name or user.username, document_count=len(documents),
|
||
paragraph_count=len(paragraphs), vector_count=len(vectors), search_ready=search_ready,
|
||
content_hash=digest, config=config)
|
||
root.version_sequence = number
|
||
root.save(update_fields=['version_sequence', 'update_time'])
|
||
return version
|
||
|
||
|
||
def get_version(workspace, knowledge_id, version_id, lock=False):
|
||
root_knowledge(workspace, knowledge_id)
|
||
query = KnowledgeVersion.objects.filter(knowledge_id=knowledge_id, knowledge__workspace_id=workspace, pk=version_id)
|
||
if lock:
|
||
query = query.select_for_update()
|
||
obj = query.first()
|
||
if obj is None:
|
||
raise NotFound404(404, '知识库版本不存在')
|
||
return obj
|
||
|
||
|
||
def validate_selection(selection, workspace, knowledge_ids):
|
||
if not isinstance(selection, dict) or len(selection) > 100:
|
||
raise ValidationError('版本选择无效')
|
||
permitted = {str(k) for k in knowledge_ids}
|
||
for root_id, chosen in selection.items():
|
||
if str(root_id) not in permitted:
|
||
raise ValidationError('只能为已关联的知识库选择版本')
|
||
if not chosen:
|
||
continue
|
||
if chosen == 'latest':
|
||
version = KnowledgeVersion.objects.filter(knowledge_id=root_id, knowledge__workspace_id=workspace,
|
||
search_ready=True).first()
|
||
else:
|
||
try:
|
||
version = KnowledgeVersion.objects.filter(pk=chosen, knowledge_id=root_id,
|
||
knowledge__workspace_id=workspace, search_ready=True).first()
|
||
except (ValueError, DjangoValidationError):
|
||
version = None
|
||
if version is None:
|
||
raise ValidationError('请选择已就绪且属于该知识库的版本')
|
||
|
||
|
||
def application_selection(knowledge_setting, work_flow=None):
|
||
selection = dict((knowledge_setting or {}).get('version_selection') or {})
|
||
knowledge_ids = []
|
||
for node in (work_flow or {}).get('nodes', []):
|
||
if node.get('type') != 'search-knowledge-node':
|
||
continue
|
||
data = node.get('properties', {}).get('node_data', {})
|
||
knowledge_ids.extend(data.get('knowledge_id_list') or [])
|
||
for root_id, chosen in (data.get('knowledge_setting') or {}).get('version_selection', {}).items():
|
||
if root_id in selection and selection[root_id] != chosen:
|
||
raise ValidationError('同一工作流中的知识库须使用相同版本')
|
||
selection[root_id] = chosen
|
||
return selection, knowledge_ids
|
||
|
||
|
||
@transaction.atomic
|
||
def pin_selection(selection, workspace, chat_id, pinned=None):
|
||
pinned = dict(pinned or {})
|
||
for root_id, chosen in sorted((selection or {}).items()):
|
||
if root_id in pinned or not chosen:
|
||
continue
|
||
root_knowledge(workspace, root_id, lock=True)
|
||
existing = KnowledgeVersionLease.objects.filter(chat_id=str(chat_id), version__knowledge_id=root_id,
|
||
expires_at__gt=timezone.now()).select_related('version').first()
|
||
if existing:
|
||
pinned[str(root_id)] = str(existing.version_id)
|
||
continue
|
||
query = KnowledgeVersion.objects.select_for_update().filter(knowledge_id=root_id,
|
||
knowledge__workspace_id=workspace, search_ready=True)
|
||
version = query.first() if chosen == 'latest' else query.filter(pk=chosen).first()
|
||
if version is None:
|
||
raise ValidationError('选定知识库版本不可用,请检查助手设置')
|
||
pinned[str(root_id)] = str(version.id)
|
||
for root_id, version_id in sorted(pinned.items()):
|
||
root_knowledge(workspace, root_id, lock=True)
|
||
version = KnowledgeVersion.objects.select_for_update().filter(pk=version_id, knowledge_id=root_id,
|
||
knowledge__workspace_id=workspace, search_ready=True).first()
|
||
if version is None:
|
||
raise ValidationError('会话使用的知识库版本已删除,请新建会话')
|
||
KnowledgeVersionLease.objects.update_or_create(version=version, chat_id=str(chat_id),
|
||
defaults={'expires_at': timezone.now() + timedelta(minutes=35)})
|
||
return pinned
|
||
|
||
|
||
def resolve_search(knowledge_ids, pinned, workspace):
|
||
result = []
|
||
for root_id in map(str, knowledge_ids):
|
||
if root_id not in (pinned or {}):
|
||
root_knowledge(workspace, root_id)
|
||
result.append(root_id)
|
||
continue
|
||
version = get_version(workspace, root_id, pinned[root_id])
|
||
if not version.search_ready:
|
||
raise ValidationError('知识库版本索引不可用')
|
||
identity = version.config.get('model_identity')
|
||
model = version.snapshot.embedding_model
|
||
if not model or (identity and identity != {'provider': model.provider, 'model_name': model.model_name,
|
||
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}):
|
||
raise ValidationError('历史版本的向量模型配置已变更,请恢复原模型配置')
|
||
result.append(str(version.snapshot_id))
|
||
return result
|
||
|
||
|
||
def resolve_document_scope(document_ids, pinned, workspace):
|
||
if document_ids is None:
|
||
return None
|
||
mapping = {}
|
||
for root_id, version_id in (pinned or {}).items():
|
||
mapping.update(get_version(workspace, root_id, version_id).config.get('_document_ids', {}))
|
||
return [mapping.get(str(document_id), str(document_id)) for document_id in document_ids]
|
||
|
||
|
||
@transaction.atomic
|
||
def delete_version(workspace, knowledge_id, version_id):
|
||
root = root_knowledge(workspace, knowledge_id, lock=True)
|
||
version = get_version(workspace, knowledge_id, version_id, lock=True)
|
||
from application.models import Application, ApplicationVersion
|
||
key = 'knowledge_setting__version_selection__' + str(root.id)
|
||
is_latest = KnowledgeVersion.objects.filter(knowledge=root, search_ready=True).first()
|
||
# Public chat uses only the latest publication. Superseded application history
|
||
# must not prevent users from retiring an otherwise unused knowledge version.
|
||
publications = ApplicationVersion.objects.filter(pk=Subquery(ApplicationVersion.objects.filter(
|
||
application_id=OuterRef('application_id')).order_by('-create_time', '-id').values('pk')[:1]))
|
||
for query in (Application.objects.all(), publications):
|
||
refs = Q(**{key: str(version.id)})
|
||
if is_latest and is_latest.id == version.id:
|
||
refs |= Q(**{key: 'latest'})
|
||
if query.filter(refs).exists():
|
||
raise ValidationError('版本已被助手或已发布配置使用,请先更换版本并重新发布')
|
||
for flow in query.filter(type='WORK_FLOW').values_list('work_flow', flat=True).iterator():
|
||
for node in (flow or {}).get('nodes', []):
|
||
chosen = ((node.get('properties', {}).get('node_data', {}).get('knowledge_setting') or {})
|
||
.get('version_selection') or {}).get(str(root.id))
|
||
if chosen == str(version.id) or (chosen == 'latest' and is_latest and is_latest.id == version.id):
|
||
raise ValidationError('版本已被工作流或已发布配置使用,请先更换版本并重新发布')
|
||
KnowledgeVersionLease.objects.filter(version=version, expires_at__lte=timezone.now()).delete()
|
||
if version.leases.exists():
|
||
raise ValidationError('版本有活跃会话,请会话结束后再删除')
|
||
snapshot_id = version.snapshot_id
|
||
Embedding.objects.filter(knowledge_id=snapshot_id).delete()
|
||
ProblemParagraphMapping.objects.filter(knowledge_id=snapshot_id).delete()
|
||
DocumentTag.objects.filter(document__knowledge_id=snapshot_id).delete()
|
||
Tag.objects.filter(knowledge_id=snapshot_id).delete()
|
||
Paragraph.objects.filter(knowledge_id=snapshot_id).delete()
|
||
Document.objects.filter(knowledge_id=snapshot_id).delete()
|
||
Problem.objects.filter(knowledge_id=snapshot_id).delete()
|
||
File.objects.filter(source_id=str(snapshot_id)).delete()
|
||
if connection.vendor == 'postgresql':
|
||
with connection.cursor() as cursor:
|
||
cursor.execute('DROP INDEX IF EXISTS "embedding_hnsw_idx_' + str(snapshot_id) + '"')
|
||
version.delete()
|
||
Knowledge.objects.filter(pk=snapshot_id).delete()
|
||
|
||
|
||
def guard_snapshot_request(request):
|
||
match = getattr(request, 'resolver_match', None)
|
||
kwargs = match.kwargs if match else {}
|
||
knowledge_ids = [kwargs[key] for key in ('knowledge_id', 'target_knowledge_id') if kwargs.get(key)]
|
||
if knowledge_ids and Knowledge.objects.filter(pk__in=knowledge_ids, meta__has_key=SNAPSHOT_KEY).exists():
|
||
raise AppUnauthorizedFailed(403, '历史版本只读,请通过原知识库的版本管理查看')
|
||
file_id = kwargs.get('file_id')
|
||
if file_id and request.method not in ('GET', 'HEAD', 'OPTIONS') and File.objects.filter(
|
||
pk=file_id, meta__has_key='_version_id').exists():
|
||
raise AppUnauthorizedFailed(403, '历史版本文件不能单独删除')
|