"""Version snapshots reuse existing retrieval storage without exposing another knowledge base.""" import copy import hashlib import json from datetime import timedelta import uuid_utils.compat as uuid from django.db import connection, transaction from django.db.models import Q, OuterRef, Subquery from django.core.exceptions import ValidationError as DjangoValidationError from django.utils import timezone from rest_framework.exceptions import ValidationError from common.exception.app_exception import AppUnauthorizedFailed, NotFound404 from knowledge.models import (Knowledge, KnowledgeVersion, KnowledgeVersionLease, Document, Paragraph, Problem, ProblemParagraphMapping, Tag, DocumentTag, Embedding, File, TaskType, State, Status) SNAPSHOT_KEY = '_version_snapshot' def root_knowledge(workspace, knowledge_id, lock=False): query = Knowledge.objects.filter(pk=knowledge_id, workspace_id=workspace).exclude(meta__has_key=SNAPSHOT_KEY) if lock: query = query.select_for_update() obj = query.first() if obj is None: raise NotFound404(404, '知识库不存在') return obj def version_data(version): return {'id': str(version.id), 'number': version.number, 'name': version.name, 'description': version.description, 'creator_name': version.creator_name, 'create_time': version.create_time, 'document_count': version.document_count, 'paragraph_count': version.paragraph_count, 'vector_count': version.vector_count, 'search_ready': version.search_ready, 'content_hash': version.content_hash, 'config': {key: value for key, value in version.config.items() if not key.startswith('_') and key != 'model_identity'}} def remap(value, mapping): if isinstance(value, dict): return {k: remap(v, mapping) for k, v in value.items()} if isinstance(value, list): return [remap(v, mapping) for v in value] if isinstance(value, str): if value in mapping: return mapping[value] # File links are embedded in Markdown, HTML and document metadata. import re return re.sub(r'(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', lambda m: mapping.get(m[0], m[0]), value) return value def copy_rows(model, source_rows, ids, overrides=None): objects = [] for row in source_rows: values = {field.attname: copy.deepcopy(getattr(row, field.attname)) for field in model._meta.concrete_fields} for field in model._meta.concrete_fields: key = field.attname value = values[key] if key.endswith('_id') or key == 'id': values[key] = ids.get(str(value), value) elif key in ('content', 'title', 'name', 'desc') and isinstance(value, str): import re values[key] = re.sub(r'(oss/file/)([0-9a-fA-F-]{36})', lambda match: match[1] + str(ids.get(match[2], match[2])), value) elif isinstance(value, (dict, list, str)): values[key] = remap(value, ids) values.update(overrides or {}) objects.append(model(**values)) return model.objects.bulk_create(objects, batch_size=500) def _build_index(snapshot_id): if connection.vendor != 'postgresql': return with connection.cursor() as cursor: cursor.execute('SELECT vector_dims(embedding) FROM embedding WHERE knowledge_id=%s LIMIT 1', [str(snapshot_id)]) row = cursor.fetchone() if row: # Identifier and predicate are generated UUIDs; dimension comes from pgvector. name = 'embedding_hnsw_idx_' + str(snapshot_id) cursor.execute('CREATE INDEX "' + name + '" ON embedding USING hnsw ' '((embedding::vector(' + str(int(row[0])) + ')) vector_cosine_ops) ' "WHERE knowledge_id='" + str(snapshot_id) + "'::uuid") def create_version(workspace, knowledge_id, user, data): name = str(data.get('name') or '').strip() description = str(data.get('description') or '').strip() if not name or len(name) > 128 or len(description) > 512: raise ValidationError('版本名称须为1~128字,说明最多512字') nested = connection.in_atomic_block with transaction.atomic(): if connection.vendor == 'postgresql': with connection.cursor() as cursor: if nested: cursor.execute('SHOW transaction_isolation') if cursor.fetchone()[0] not in ('repeatable read', 'serializable'): raise ValidationError('版本创建需使用独立的一致性事务') else: cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ') root = root_knowledge(workspace, knowledge_id, lock=True) documents = list(Document.objects.filter(knowledge=root).order_by('id')) if any(Status(doc.status)[kind] in (State.PENDING, State.STARTED) for doc in documents for kind in (TaskType.EMBEDDING, TaskType.GENERATE_PROBLEM, TaskType.SYNC)): raise ValidationError('文档正在处理,请完成后创建版本') paragraphs = list(Paragraph.objects.filter(knowledge=root).order_by('id')) problems = list(Problem.objects.filter(knowledge=root).order_by('id')) mappings = list(ProblemParagraphMapping.objects.filter(knowledge=root)) tags = list(Tag.objects.filter(knowledge=root)) doc_tags = list(DocumentTag.objects.filter(document__knowledge=root)) vectors = list(Embedding.objects.filter(knowledge=root)) files = list(File.objects.filter(Q(source_id=str(root.id)) | Q(source_id__in=[str(d.id) for d in documents]) | Q(meta__knowledge_id=str(root.id)))) number = root.version_sequence + 1 latest = KnowledgeVersion.objects.filter(knowledge=root).first() number = max(number, latest.number + 1 if latest else 1) version_id, snapshot_id = uuid.uuid7(), uuid.uuid7() ids = {str(root.id): str(snapshot_id)} for rows in (documents, paragraphs, problems, mappings, tags, doc_tags, vectors, files): ids.update({str(row.pk): str(uuid.uuid7()) for row in rows}) config = {'embedding_model_id': str(root.embedding_model_id) if root.embedding_model_id else None, 'embedding_params': (root.meta or {}).get('embedding_params', {}), 'segmentation': (root.meta or {}).get('segmentation', {}), 'knowledge_name': root.name, 'knowledge_type': root.type, 'metadata': (root.meta or {}).get('business_metadata', {})} model = root.embedding_model if model: config['model_identity'] = {'provider': model.provider, 'model_name': model.model_name, 'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()} content = {'documents': [[str(d.id), d.name, d.is_active, d.meta] for d in documents], 'paragraphs': [[str(p.id), str(p.document_id), p.position, p.title, p.content, p.is_active] for p in paragraphs], 'problems': [[str(p.id), p.content] for p in problems], 'mappings': [[str(p.problem_id), str(p.paragraph_id)] for p in mappings], 'config': config} digest = hashlib.sha256(json.dumps(content, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest() config['_document_ids'] = {str(d.id): ids[str(d.id)] for d in documents} metadata = copy.deepcopy(root.meta or {}) metadata.pop('_version_sequence', None) metadata.pop('parser_id', None) metadata.update({SNAPSHOT_KEY: str(root.id), '_version_id': str(version_id)}) snapshot = Knowledge.objects.create(id=snapshot_id, name=(root.name[:125] + ' · v' + str(number))[:150], desc=root.desc, workspace_id=root.workspace_id, folder_id=root.folder_id, user_id=root.user_id, embedding_model_id=root.embedding_model_id, scope=root.scope, type=0, file_size_limit=root.file_size_limit, file_count_limit=root.file_count_limit, meta=metadata) copy_rows(Document, documents, ids, {'knowledge_id': snapshot_id, 'type': 0}) copy_rows(Paragraph, paragraphs, ids, {'knowledge_id': snapshot_id, 'hit_num': 0}) copy_rows(Problem, problems, ids, {'knowledge_id': snapshot_id, 'hit_num': 0}) copy_rows(ProblemParagraphMapping, mappings, ids, {'knowledge_id': snapshot_id}) copy_rows(Tag, tags, ids, {'knowledge_id': snapshot_id}) copy_rows(DocumentTag, doc_tags, ids) # Preserve binary large-object references; only the last File reference frees the bytes. cloned_files = copy_rows(File, files, ids, {'source_id': str(snapshot_id)}) for file in cloned_files: file.meta = {**(file.meta or {}), '_version_id': str(version_id), 'knowledge_id': str(root.id)} File.objects.bulk_update(cloned_files, ['meta'], batch_size=500) copied_vectors = copy_rows(Embedding, vectors, ids, {'knowledge_id': snapshot_id}) for old, new in zip(vectors, copied_vectors): new.source_id = ids.get(str(old.source_id), old.source_id) Embedding.objects.bulk_update(copied_vectors, ['source_id'], batch_size=500) active_docs = {d.id for d in documents if d.is_active} active_paragraphs = {p.id for p in paragraphs if p.is_active and p.document_id in active_docs} indexed = {v.paragraph_id for v in vectors if v.is_active} search_ready = bool(model and active_paragraphs and active_paragraphs <= indexed) _build_index(snapshot_id) version = KnowledgeVersion.objects.create(id=version_id, knowledge=root, snapshot=snapshot, number=number, name=name, description=description, creator=user, creator_name=user.nick_name or user.username, document_count=len(documents), paragraph_count=len(paragraphs), vector_count=len(vectors), search_ready=search_ready, content_hash=digest, config=config) root.version_sequence = number root.save(update_fields=['version_sequence', 'update_time']) return version def get_version(workspace, knowledge_id, version_id, lock=False): root_knowledge(workspace, knowledge_id) query = KnowledgeVersion.objects.filter(knowledge_id=knowledge_id, knowledge__workspace_id=workspace, pk=version_id) if lock: query = query.select_for_update() obj = query.first() if obj is None: raise NotFound404(404, '知识库版本不存在') return obj def validate_selection(selection, workspace, knowledge_ids): if not isinstance(selection, dict) or len(selection) > 100: raise ValidationError('版本选择无效') permitted = {str(k) for k in knowledge_ids} for root_id, chosen in selection.items(): if str(root_id) not in permitted: raise ValidationError('只能为已关联的知识库选择版本') if not chosen: continue if chosen == 'latest': version = KnowledgeVersion.objects.filter(knowledge_id=root_id, knowledge__workspace_id=workspace, search_ready=True).first() else: try: version = KnowledgeVersion.objects.filter(pk=chosen, knowledge_id=root_id, knowledge__workspace_id=workspace, search_ready=True).first() except (ValueError, DjangoValidationError): version = None if version is None: raise ValidationError('请选择已就绪且属于该知识库的版本') def application_selection(knowledge_setting, work_flow=None): selection = dict((knowledge_setting or {}).get('version_selection') or {}) knowledge_ids = [] for node in (work_flow or {}).get('nodes', []): if node.get('type') != 'search-knowledge-node': continue data = node.get('properties', {}).get('node_data', {}) knowledge_ids.extend(data.get('knowledge_id_list') or []) for root_id, chosen in (data.get('knowledge_setting') or {}).get('version_selection', {}).items(): if root_id in selection and selection[root_id] != chosen: raise ValidationError('同一工作流中的知识库须使用相同版本') selection[root_id] = chosen return selection, knowledge_ids @transaction.atomic def pin_selection(selection, workspace, chat_id, pinned=None): pinned = dict(pinned or {}) for root_id, chosen in sorted((selection or {}).items()): if root_id in pinned or not chosen: continue root_knowledge(workspace, root_id, lock=True) existing = KnowledgeVersionLease.objects.filter(chat_id=str(chat_id), version__knowledge_id=root_id, expires_at__gt=timezone.now()).select_related('version').first() if existing: pinned[str(root_id)] = str(existing.version_id) continue query = KnowledgeVersion.objects.select_for_update().filter(knowledge_id=root_id, knowledge__workspace_id=workspace, search_ready=True) version = query.first() if chosen == 'latest' else query.filter(pk=chosen).first() if version is None: raise ValidationError('选定知识库版本不可用,请检查助手设置') pinned[str(root_id)] = str(version.id) for root_id, version_id in sorted(pinned.items()): root_knowledge(workspace, root_id, lock=True) version = KnowledgeVersion.objects.select_for_update().filter(pk=version_id, knowledge_id=root_id, knowledge__workspace_id=workspace, search_ready=True).first() if version is None: raise ValidationError('会话使用的知识库版本已删除,请新建会话') KnowledgeVersionLease.objects.update_or_create(version=version, chat_id=str(chat_id), defaults={'expires_at': timezone.now() + timedelta(minutes=35)}) return pinned def resolve_search(knowledge_ids, pinned, workspace): result = [] for root_id in map(str, knowledge_ids): if root_id not in (pinned or {}): root_knowledge(workspace, root_id) result.append(root_id) continue version = get_version(workspace, root_id, pinned[root_id]) if not version.search_ready: raise ValidationError('知识库版本索引不可用') identity = version.config.get('model_identity') model = version.snapshot.embedding_model if not model or (identity and identity != {'provider': model.provider, 'model_name': model.model_name, 'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}): raise ValidationError('历史版本的向量模型配置已变更,请恢复原模型配置') result.append(str(version.snapshot_id)) return result def resolve_document_scope(document_ids, pinned, workspace): if document_ids is None: return None mapping = {} for root_id, version_id in (pinned or {}).items(): mapping.update(get_version(workspace, root_id, version_id).config.get('_document_ids', {})) return [mapping.get(str(document_id), str(document_id)) for document_id in document_ids] @transaction.atomic def delete_version(workspace, knowledge_id, version_id): root = root_knowledge(workspace, knowledge_id, lock=True) version = get_version(workspace, knowledge_id, version_id, lock=True) from application.models import Application, ApplicationVersion key = 'knowledge_setting__version_selection__' + str(root.id) is_latest = KnowledgeVersion.objects.filter(knowledge=root, search_ready=True).first() # Public chat uses only the latest publication. Superseded application history # must not prevent users from retiring an otherwise unused knowledge version. publications = ApplicationVersion.objects.filter(pk=Subquery(ApplicationVersion.objects.filter( application_id=OuterRef('application_id')).order_by('-create_time', '-id').values('pk')[:1])) for query in (Application.objects.all(), publications): refs = Q(**{key: str(version.id)}) if is_latest and is_latest.id == version.id: refs |= Q(**{key: 'latest'}) if query.filter(refs).exists(): raise ValidationError('版本已被助手或已发布配置使用,请先更换版本并重新发布') for flow in query.filter(type='WORK_FLOW').values_list('work_flow', flat=True).iterator(): for node in (flow or {}).get('nodes', []): chosen = ((node.get('properties', {}).get('node_data', {}).get('knowledge_setting') or {}) .get('version_selection') or {}).get(str(root.id)) if chosen == str(version.id) or (chosen == 'latest' and is_latest and is_latest.id == version.id): raise ValidationError('版本已被工作流或已发布配置使用,请先更换版本并重新发布') KnowledgeVersionLease.objects.filter(version=version, expires_at__lte=timezone.now()).delete() if version.leases.exists(): raise ValidationError('版本有活跃会话,请会话结束后再删除') snapshot_id = version.snapshot_id Embedding.objects.filter(knowledge_id=snapshot_id).delete() ProblemParagraphMapping.objects.filter(knowledge_id=snapshot_id).delete() DocumentTag.objects.filter(document__knowledge_id=snapshot_id).delete() Tag.objects.filter(knowledge_id=snapshot_id).delete() Paragraph.objects.filter(knowledge_id=snapshot_id).delete() Document.objects.filter(knowledge_id=snapshot_id).delete() Problem.objects.filter(knowledge_id=snapshot_id).delete() File.objects.filter(source_id=str(snapshot_id)).delete() if connection.vendor == 'postgresql': with connection.cursor() as cursor: cursor.execute('DROP INDEX IF EXISTS "embedding_hnsw_idx_' + str(snapshot_id) + '"') version.delete() Knowledge.objects.filter(pk=snapshot_id).delete() def guard_snapshot_request(request): match = getattr(request, 'resolver_match', None) kwargs = match.kwargs if match else {} knowledge_ids = [kwargs[key] for key in ('knowledge_id', 'target_knowledge_id') if kwargs.get(key)] if knowledge_ids and Knowledge.objects.filter(pk__in=knowledge_ids, meta__has_key=SNAPSHOT_KEY).exists(): raise AppUnauthorizedFailed(403, '历史版本只读,请通过原知识库的版本管理查看') file_id = kwargs.get('file_id') if file_id and request.method not in ('GET', 'HEAD', 'OPTIONS') and File.objects.filter( pk=file_id, meta__has_key='_version_id').exists(): raise AppUnauthorizedFailed(403, '历史版本文件不能单独删除')