UnisKB/apps/knowledge/versions.py

332 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Version snapshots reuse existing retrieval storage without exposing another knowledge base."""
import copy
import hashlib
import json
from datetime import timedelta
import uuid_utils.compat as uuid
from django.db import connection, transaction
from django.db.models import Q, OuterRef, Subquery
from django.core.exceptions import ValidationError as DjangoValidationError
from django.utils import timezone
from rest_framework.exceptions import ValidationError
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
from knowledge.models import (Knowledge, KnowledgeVersion, KnowledgeVersionLease, Document, Paragraph,
Problem, ProblemParagraphMapping, Tag, DocumentTag, Embedding, File, TaskType, State, Status)
SNAPSHOT_KEY = '_version_snapshot'
def root_knowledge(workspace, knowledge_id, lock=False):
query = Knowledge.objects.filter(pk=knowledge_id, workspace_id=workspace).exclude(meta__has_key=SNAPSHOT_KEY)
if lock:
query = query.select_for_update()
obj = query.first()
if obj is None:
raise NotFound404(404, '知识库不存在')
return obj
def version_data(version):
return {'id': str(version.id), 'number': version.number, 'name': version.name,
'description': version.description, 'creator_name': version.creator_name,
'create_time': version.create_time, 'document_count': version.document_count,
'paragraph_count': version.paragraph_count, 'vector_count': version.vector_count,
'search_ready': version.search_ready, 'content_hash': version.content_hash,
'config': {key: value for key, value in version.config.items()
if not key.startswith('_') and key != 'model_identity'}}
def remap(value, mapping):
if isinstance(value, dict):
return {k: remap(v, mapping) for k, v in value.items()}
if isinstance(value, list):
return [remap(v, mapping) for v in value]
if isinstance(value, str):
if value in mapping:
return mapping[value]
# File links are embedded in Markdown, HTML and document metadata.
import re
return re.sub(r'(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
lambda m: mapping.get(m[0], m[0]), value)
return value
def copy_rows(model, source_rows, ids, overrides=None):
objects = []
for row in source_rows:
values = {field.attname: copy.deepcopy(getattr(row, field.attname)) for field in model._meta.concrete_fields}
for field in model._meta.concrete_fields:
key = field.attname
value = values[key]
if key.endswith('_id') or key == 'id':
values[key] = ids.get(str(value), value)
elif key in ('content', 'title', 'name', 'desc') and isinstance(value, str):
import re
values[key] = re.sub(r'(oss/file/)([0-9a-fA-F-]{36})',
lambda match: match[1] + str(ids.get(match[2], match[2])), value)
elif isinstance(value, (dict, list, str)):
values[key] = remap(value, ids)
values.update(overrides or {})
objects.append(model(**values))
return model.objects.bulk_create(objects, batch_size=500)
def _build_index(snapshot_id):
if connection.vendor != 'postgresql':
return
with connection.cursor() as cursor:
cursor.execute('SELECT vector_dims(embedding) FROM embedding WHERE knowledge_id=%s LIMIT 1', [str(snapshot_id)])
row = cursor.fetchone()
if row:
# Identifier and predicate are generated UUIDs; dimension comes from pgvector.
name = 'embedding_hnsw_idx_' + str(snapshot_id)
cursor.execute('CREATE INDEX "' + name + '" ON embedding USING hnsw '
'((embedding::vector(' + str(int(row[0])) + ')) vector_cosine_ops) '
"WHERE knowledge_id='" + str(snapshot_id) + "'::uuid")
def create_version(workspace, knowledge_id, user, data):
name = str(data.get('name') or '').strip()
description = str(data.get('description') or '').strip()
if not name or len(name) > 128 or len(description) > 512:
raise ValidationError('版本名称须为1~128字,说明最多512字')
nested = connection.in_atomic_block
with transaction.atomic():
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
if nested:
cursor.execute('SHOW transaction_isolation')
if cursor.fetchone()[0] not in ('repeatable read', 'serializable'):
raise ValidationError('版本创建需使用独立的一致性事务')
else:
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
root = root_knowledge(workspace, knowledge_id, lock=True)
documents = list(Document.objects.filter(knowledge=root).order_by('id'))
if any(Status(doc.status)[kind] in (State.PENDING, State.STARTED)
for doc in documents for kind in (TaskType.EMBEDDING, TaskType.GENERATE_PROBLEM, TaskType.SYNC)):
raise ValidationError('文档正在处理,请完成后创建版本')
paragraphs = list(Paragraph.objects.filter(knowledge=root).order_by('id'))
problems = list(Problem.objects.filter(knowledge=root).order_by('id'))
mappings = list(ProblemParagraphMapping.objects.filter(knowledge=root))
tags = list(Tag.objects.filter(knowledge=root))
doc_tags = list(DocumentTag.objects.filter(document__knowledge=root))
vectors = list(Embedding.objects.filter(knowledge=root))
files = list(File.objects.filter(Q(source_id=str(root.id)) |
Q(source_id__in=[str(d.id) for d in documents]) | Q(meta__knowledge_id=str(root.id))))
number = root.version_sequence + 1
latest = KnowledgeVersion.objects.filter(knowledge=root).first()
number = max(number, latest.number + 1 if latest else 1)
version_id, snapshot_id = uuid.uuid7(), uuid.uuid7()
ids = {str(root.id): str(snapshot_id)}
for rows in (documents, paragraphs, problems, mappings, tags, doc_tags, vectors, files):
ids.update({str(row.pk): str(uuid.uuid7()) for row in rows})
config = {'embedding_model_id': str(root.embedding_model_id) if root.embedding_model_id else None,
'embedding_params': (root.meta or {}).get('embedding_params', {}),
'segmentation': (root.meta or {}).get('segmentation', {}), 'knowledge_name': root.name,
'knowledge_type': root.type, 'metadata': (root.meta or {}).get('business_metadata', {})}
model = root.embedding_model
if model:
config['model_identity'] = {'provider': model.provider, 'model_name': model.model_name,
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}
content = {'documents': [[str(d.id), d.name, d.is_active, d.meta] for d in documents],
'paragraphs': [[str(p.id), str(p.document_id), p.position, p.title, p.content, p.is_active] for p in paragraphs],
'problems': [[str(p.id), p.content] for p in problems],
'mappings': [[str(p.problem_id), str(p.paragraph_id)] for p in mappings], 'config': config}
digest = hashlib.sha256(json.dumps(content, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest()
config['_document_ids'] = {str(d.id): ids[str(d.id)] for d in documents}
metadata = copy.deepcopy(root.meta or {})
metadata.pop('_version_sequence', None)
metadata.pop('parser_id', None)
metadata.update({SNAPSHOT_KEY: str(root.id), '_version_id': str(version_id)})
snapshot = Knowledge.objects.create(id=snapshot_id, name=(root.name[:125] + ' · v' + str(number))[:150],
desc=root.desc, workspace_id=root.workspace_id, folder_id=root.folder_id, user_id=root.user_id,
embedding_model_id=root.embedding_model_id, scope=root.scope, type=0,
file_size_limit=root.file_size_limit, file_count_limit=root.file_count_limit, meta=metadata)
copy_rows(Document, documents, ids, {'knowledge_id': snapshot_id, 'type': 0})
copy_rows(Paragraph, paragraphs, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
copy_rows(Problem, problems, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
copy_rows(ProblemParagraphMapping, mappings, ids, {'knowledge_id': snapshot_id})
copy_rows(Tag, tags, ids, {'knowledge_id': snapshot_id})
copy_rows(DocumentTag, doc_tags, ids)
# Preserve binary large-object references; only the last File reference frees the bytes.
cloned_files = copy_rows(File, files, ids, {'source_id': str(snapshot_id)})
for file in cloned_files:
file.meta = {**(file.meta or {}), '_version_id': str(version_id), 'knowledge_id': str(root.id)}
File.objects.bulk_update(cloned_files, ['meta'], batch_size=500)
copied_vectors = copy_rows(Embedding, vectors, ids, {'knowledge_id': snapshot_id})
for old, new in zip(vectors, copied_vectors):
new.source_id = ids.get(str(old.source_id), old.source_id)
Embedding.objects.bulk_update(copied_vectors, ['source_id'], batch_size=500)
active_docs = {d.id for d in documents if d.is_active}
active_paragraphs = {p.id for p in paragraphs if p.is_active and p.document_id in active_docs}
indexed = {v.paragraph_id for v in vectors if v.is_active}
search_ready = bool(model and active_paragraphs and active_paragraphs <= indexed)
_build_index(snapshot_id)
version = KnowledgeVersion.objects.create(id=version_id, knowledge=root, snapshot=snapshot,
number=number, name=name, description=description, creator=user,
creator_name=user.nick_name or user.username, document_count=len(documents),
paragraph_count=len(paragraphs), vector_count=len(vectors), search_ready=search_ready,
content_hash=digest, config=config)
root.version_sequence = number
root.save(update_fields=['version_sequence', 'update_time'])
return version
def get_version(workspace, knowledge_id, version_id, lock=False):
root_knowledge(workspace, knowledge_id)
query = KnowledgeVersion.objects.filter(knowledge_id=knowledge_id, knowledge__workspace_id=workspace, pk=version_id)
if lock:
query = query.select_for_update()
obj = query.first()
if obj is None:
raise NotFound404(404, '知识库版本不存在')
return obj
def validate_selection(selection, workspace, knowledge_ids):
if not isinstance(selection, dict) or len(selection) > 100:
raise ValidationError('版本选择无效')
permitted = {str(k) for k in knowledge_ids}
for root_id, chosen in selection.items():
if str(root_id) not in permitted:
raise ValidationError('只能为已关联的知识库选择版本')
if not chosen:
continue
if chosen == 'latest':
version = KnowledgeVersion.objects.filter(knowledge_id=root_id, knowledge__workspace_id=workspace,
search_ready=True).first()
else:
try:
version = KnowledgeVersion.objects.filter(pk=chosen, knowledge_id=root_id,
knowledge__workspace_id=workspace, search_ready=True).first()
except (ValueError, DjangoValidationError):
version = None
if version is None:
raise ValidationError('请选择已就绪且属于该知识库的版本')
def application_selection(knowledge_setting, work_flow=None):
selection = dict((knowledge_setting or {}).get('version_selection') or {})
knowledge_ids = []
for node in (work_flow or {}).get('nodes', []):
if node.get('type') != 'search-knowledge-node':
continue
data = node.get('properties', {}).get('node_data', {})
knowledge_ids.extend(data.get('knowledge_id_list') or [])
for root_id, chosen in (data.get('knowledge_setting') or {}).get('version_selection', {}).items():
if root_id in selection and selection[root_id] != chosen:
raise ValidationError('同一工作流中的知识库须使用相同版本')
selection[root_id] = chosen
return selection, knowledge_ids
@transaction.atomic
def pin_selection(selection, workspace, chat_id, pinned=None):
pinned = dict(pinned or {})
for root_id, chosen in sorted((selection or {}).items()):
if root_id in pinned or not chosen:
continue
root_knowledge(workspace, root_id, lock=True)
existing = KnowledgeVersionLease.objects.filter(chat_id=str(chat_id), version__knowledge_id=root_id,
expires_at__gt=timezone.now()).select_related('version').first()
if existing:
pinned[str(root_id)] = str(existing.version_id)
continue
query = KnowledgeVersion.objects.select_for_update().filter(knowledge_id=root_id,
knowledge__workspace_id=workspace, search_ready=True)
version = query.first() if chosen == 'latest' else query.filter(pk=chosen).first()
if version is None:
raise ValidationError('选定知识库版本不可用,请检查助手设置')
pinned[str(root_id)] = str(version.id)
for root_id, version_id in sorted(pinned.items()):
root_knowledge(workspace, root_id, lock=True)
version = KnowledgeVersion.objects.select_for_update().filter(pk=version_id, knowledge_id=root_id,
knowledge__workspace_id=workspace, search_ready=True).first()
if version is None:
raise ValidationError('会话使用的知识库版本已删除,请新建会话')
KnowledgeVersionLease.objects.update_or_create(version=version, chat_id=str(chat_id),
defaults={'expires_at': timezone.now() + timedelta(minutes=35)})
return pinned
def resolve_search(knowledge_ids, pinned, workspace):
result = []
for root_id in map(str, knowledge_ids):
if root_id not in (pinned or {}):
root_knowledge(workspace, root_id)
result.append(root_id)
continue
version = get_version(workspace, root_id, pinned[root_id])
if not version.search_ready:
raise ValidationError('知识库版本索引不可用')
identity = version.config.get('model_identity')
model = version.snapshot.embedding_model
if not model or (identity and identity != {'provider': model.provider, 'model_name': model.model_name,
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}):
raise ValidationError('历史版本的向量模型配置已变更,请恢复原模型配置')
result.append(str(version.snapshot_id))
return result
def resolve_document_scope(document_ids, pinned, workspace):
if document_ids is None:
return None
mapping = {}
for root_id, version_id in (pinned or {}).items():
mapping.update(get_version(workspace, root_id, version_id).config.get('_document_ids', {}))
return [mapping.get(str(document_id), str(document_id)) for document_id in document_ids]
@transaction.atomic
def delete_version(workspace, knowledge_id, version_id):
root = root_knowledge(workspace, knowledge_id, lock=True)
version = get_version(workspace, knowledge_id, version_id, lock=True)
from application.models import Application, ApplicationVersion
key = 'knowledge_setting__version_selection__' + str(root.id)
is_latest = KnowledgeVersion.objects.filter(knowledge=root, search_ready=True).first()
# Public chat uses only the latest publication. Superseded application history
# must not prevent users from retiring an otherwise unused knowledge version.
publications = ApplicationVersion.objects.filter(pk=Subquery(ApplicationVersion.objects.filter(
application_id=OuterRef('application_id')).order_by('-create_time', '-id').values('pk')[:1]))
for query in (Application.objects.all(), publications):
refs = Q(**{key: str(version.id)})
if is_latest and is_latest.id == version.id:
refs |= Q(**{key: 'latest'})
if query.filter(refs).exists():
raise ValidationError('版本已被助手或已发布配置使用,请先更换版本并重新发布')
for flow in query.filter(type='WORK_FLOW').values_list('work_flow', flat=True).iterator():
for node in (flow or {}).get('nodes', []):
chosen = ((node.get('properties', {}).get('node_data', {}).get('knowledge_setting') or {})
.get('version_selection') or {}).get(str(root.id))
if chosen == str(version.id) or (chosen == 'latest' and is_latest and is_latest.id == version.id):
raise ValidationError('版本已被工作流或已发布配置使用,请先更换版本并重新发布')
KnowledgeVersionLease.objects.filter(version=version, expires_at__lte=timezone.now()).delete()
if version.leases.exists():
raise ValidationError('版本有活跃会话,请会话结束后再删除')
snapshot_id = version.snapshot_id
Embedding.objects.filter(knowledge_id=snapshot_id).delete()
ProblemParagraphMapping.objects.filter(knowledge_id=snapshot_id).delete()
DocumentTag.objects.filter(document__knowledge_id=snapshot_id).delete()
Tag.objects.filter(knowledge_id=snapshot_id).delete()
Paragraph.objects.filter(knowledge_id=snapshot_id).delete()
Document.objects.filter(knowledge_id=snapshot_id).delete()
Problem.objects.filter(knowledge_id=snapshot_id).delete()
File.objects.filter(source_id=str(snapshot_id)).delete()
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
cursor.execute('DROP INDEX IF EXISTS "embedding_hnsw_idx_' + str(snapshot_id) + '"')
version.delete()
Knowledge.objects.filter(pk=snapshot_id).delete()
def guard_snapshot_request(request):
match = getattr(request, 'resolver_match', None)
kwargs = match.kwargs if match else {}
knowledge_ids = [kwargs[key] for key in ('knowledge_id', 'target_knowledge_id') if kwargs.get(key)]
if knowledge_ids and Knowledge.objects.filter(pk__in=knowledge_ids, meta__has_key=SNAPSHOT_KEY).exists():
raise AppUnauthorizedFailed(403, '历史版本只读,请通过原知识库的版本管理查看')
file_id = kwargs.get('file_id')
if file_id and request.method not in ('GET', 'HEAD', 'OPTIONS') and File.objects.filter(
pk=file_id, meta__has_key='_version_id').exists():
raise AppUnauthorizedFailed(403, '历史版本文件不能单独删除')