From 2baba0b19c736a264641646bf3308560bae5200e Mon Sep 17 00:00:00 2001 From: panyy Date: Tue, 27 Oct 2026 13:37:24 +0800 Subject: [PATCH] =?UTF-8?q?feat:=E7=9F=A5=E8=AF=86=E5=BA=93=E7=89=88?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/base_search_dataset_step.py | 6 + .../i_search_knowledge_node.py | 1 + .../impl/base_search_knowledge_node.py | 7 + apps/application/serializers/application.py | 20 ++ .../serializers/application_chat_record.py | 3 + apps/application/serializers/common.py | 24 +- apps/chat/serializers/chat.py | 9 +- apps/common/auth/authenticate.py | 2 + apps/folders/serializers/folder.py | 10 +- apps/knowledge/models/__init__.py | 1 + apps/knowledge/models/knowledge.py | 3 +- apps/knowledge/serializers/knowledge.py | 6 +- apps/knowledge/urls.py | 3 + apps/system_manage/platform_access.py | 7 +- .../check_member_permission_target_exists.sql | 2 +- ...get_knowledge_user_resource_permission.sql | 2 +- .../sql/get_user_resource_permission.sql | 2 +- .../views/application/ApplicationSetting.vue | 62 ++++- ui/src/views/document/index.vue | 23 +- ui/src/views/knowledge/KnowledgeSetting.vue | 262 +++++++++++++----- .../nodes/search-knowledge-node/index.vue | 32 ++- 21 files changed, 393 insertions(+), 94 deletions(-) diff --git a/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py b/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py index 6281ffccb..4ed80d21d 100644 --- a/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py +++ b/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py @@ -61,6 +61,12 @@ class BaseSearchDatasetStep(ISearchDatasetStep): knowledge_id_list) if len(knowledge_id_list) == 0: return [] + from knowledge.versions import resolve_search + knowledge_id_list = resolve_search(knowledge_id_list, manage.context.get('knowledge_version_map'), workspace_id) + # Exclusion IDs must come from the selected content, including frozen inactive documents. + from knowledge.models import Document + exclude_document_id_list = list(Document.objects.filter(knowledge_id__in=knowledge_id_list, + is_active=False).values_list('id', flat=True)) exec_problem_text = padding_problem_text if padding_problem_text is not None else problem_text model_id = get_embedding_id(knowledge_id_list) model = get_model_by_id(model_id, workspace_id) diff --git a/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py b/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py index 7f9311e2d..c919c523e 100644 --- a/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py +++ b/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py @@ -18,6 +18,7 @@ from common.utils.common import flat_map class DatasetSettingSerializer(serializers.Serializer): + version_selection = serializers.DictField(child=serializers.CharField(allow_blank=True), required=False) # 需要查询的条数 top_n = serializers.IntegerField(required=True, label=_("Reference segment number")) diff --git a/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py b/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py index e7a0a676e..722299c4b 100644 --- a/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py +++ b/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py @@ -101,6 +101,13 @@ class BaseSearchKnowledgeNode(ISearchKnowledgeStepNode): knowledge_id_list) if len(knowledge_id_list) == 0: return get_none_result(question) + from knowledge.versions import resolve_search, resolve_document_scope + knowledge_id_list = resolve_search(knowledge_id_list, + self.workflow_manage.get_body().get('knowledge_version_map'), + self.workflow_manage.get_body().get('workspace_id')) + document_id_list = resolve_document_scope(document_id_list, + self.workflow_manage.get_body().get('knowledge_version_map'), + self.workflow_manage.get_body().get('workspace_id')) model_id = get_embedding_id(knowledge_id_list) workspace_id = self.workflow_manage.get_body().get('workspace_id') get_model_instance_by_model_workspace_id(model_id, workspace_id) # Preserve resource access validation. diff --git a/apps/application/serializers/application.py b/apps/application/serializers/application.py index 26bc1a6e8..9b9c67424 100644 --- a/apps/application/serializers/application.py +++ b/apps/application/serializers/application.py @@ -98,6 +98,7 @@ class NoReferencesSetting(serializers.Serializer): class KnowledgeSettingSerializer(serializers.Serializer): + version_selection = serializers.DictField(child=serializers.CharField(allow_blank=True), required=False) top_n = serializers.FloatField(required=True, max_value=10000, min_value=1, label=_("Reference segment number")) similarity = serializers.FloatField(required=True, max_value=1, min_value=0, @@ -132,6 +133,8 @@ class ModelKnowledgeAssociation(serializers.Serializer): if not QuerySet(Model).filter(id=model_id).exists(): raise AppApiException(500, f'{_("Model does not exist")}【{model_id}】') knowledge_id_list = list(set(self.data.get('knowledge_id_list', []))) + if Knowledge.objects.filter(id__in=knowledge_id_list, meta__has_key='_version_snapshot').exists(): + raise AppApiException(500, '请关联原知识库,再选择历史版本') if workspace_id is not None and len(workspace_id) > 0: exist_knowledge_id_list = [ str(knowledge.get('id')) for knowledge in KnowledgeSerializer.Query(data={ @@ -272,6 +275,9 @@ class ApplicationCreateSerializer(serializers.Serializer): ModelKnowledgeAssociation(data={'user_id': user_id, 'workspace_id': workspace_id, 'model_id': self.data.get('model_id'), 'knowledge_id_list': self.data.get('knowledge_id_list')}).is_valid() + from knowledge.versions import validate_selection + validate_selection((self.data.get('knowledge_setting') or {}).get('version_selection', {}), + workspace_id, self.data.get('knowledge_id_list') or []) @staticmethod def to_application_model(user_id: str, workspace_id: str, application: Dict): @@ -780,6 +786,13 @@ class ApplicationOperateSerializer(serializers.Serializer): publish_user_name=user.username, workspace_id=workspace_id) self.reset_application_version(work_flow_version, application) + from knowledge.versions import application_selection, validate_selection + root_ids = [str(k) for k in ApplicationKnowledgeMapping.objects.filter(application=application) + .values_list('knowledge_id', flat=True)] + selected, flow_ids = application_selection(application.knowledge_setting, + application.work_flow if application.type == ApplicationTypeChoices.WORK_FLOW else None) + validate_selection(selected, workspace_id, root_ids + flow_ids) + work_flow_version.knowledge_setting = {**(work_flow_version.knowledge_setting or {}), '_knowledge_ids': root_ids} work_flow_version.save() access_token = hashlib.md5( str(uuid.uuid7()).encode()).hexdigest()[ @@ -838,6 +851,13 @@ class ApplicationOperateSerializer(serializers.Serializer): application_id = self.data.get("application_id") application = QuerySet(Application).get(id=application_id) + from knowledge.versions import validate_selection, application_selection + selected_ids = instance.get('knowledge_id_list') + if selected_ids is None: + selected_ids = list(ApplicationKnowledgeMapping.objects.filter(application_id=application_id).values_list('knowledge_id', flat=True)) + selected, flow_ids = application_selection(instance.get('knowledge_setting', application.knowledge_setting), + instance.get('work_flow', application.work_flow) if application.type == ApplicationTypeChoices.WORK_FLOW else None) + validate_selection(selected, application.workspace_id, list(selected_ids) + flow_ids) if instance.get('model_id') is None or len(instance.get('model_id')) == 0: application.model_id = None else: diff --git a/apps/application/serializers/application_chat_record.py b/apps/application/serializers/application_chat_record.py index f3064968c..1269dc16d 100644 --- a/apps/application/serializers/application_chat_record.py +++ b/apps/application/serializers/application_chat_record.py @@ -311,6 +311,9 @@ class ApplicationChatRecordAddKnowledgeSerializer(serializers.Serializer): if with_valid: ApplicationChatRecordAddKnowledgeSerializer(data=instance).is_valid(raise_exception=True) self.is_valid(raise_exception=True) + # This endpoint receives its knowledge ID in the body rather than the route. + from knowledge.versions import root_knowledge + root_knowledge(self.data.get('workspace_id') or 'default', self.data['knowledge_id']) if scope == 'WORKSPACE': is_permission = get_is_permissions(request=request, workspace_id=self.data.get('workspace_id'), knowledge_id=self.data.get("knowledge_id"))( diff --git a/apps/application/serializers/common.py b/apps/application/serializers/common.py index b85252b0f..4c920bbdf 100644 --- a/apps/application/serializers/common.py +++ b/apps/application/serializers/common.py @@ -56,6 +56,7 @@ class ChatInfo: self.application = None self.chat_user = None self.debug = debug + self.knowledge_version_map = None @staticmethod def get_no_references_setting(knowledge_setting, model_setting): @@ -75,14 +76,14 @@ class ChatInfo: raise ChatException(500, _('The application does not exist')) else: application = QuerySet(ApplicationVersion).filter(application_id=self.application_id).order_by( - '-create_time')[0:1].first() + '-create_time', '-id')[0:1].first() if not application: raise ChatException(500, _("The application has not been published. Please use it after publishing.")) if application.type == ApplicationTypeChoices.SIMPLE.value: # 数据集id列表 - knowledge_id_list = [str(row.knowledge_id) for row in - QuerySet(ApplicationKnowledgeMapping).filter( - application_id=self.application_id)] + published_ids = (application.knowledge_setting or {}).get('_knowledge_ids') if not self.debug else None + knowledge_id_list = published_ids if published_ids is not None else [str(row.knowledge_id) for row in + QuerySet(ApplicationKnowledgeMapping).filter(application_id=self.application_id)] # 需要排除的文档 exclude_document_id_list = [str(document.id) for document in @@ -92,6 +93,16 @@ class ChatInfo: self.knowledge_id_list = knowledge_id_list self.exclude_document_id_list = exclude_document_id_list self.application = application + from knowledge.versions import pin_selection, validate_selection, application_selection + selection, flow_ids = application_selection(application.knowledge_setting, + application.work_flow if application.type == ApplicationTypeChoices.WORK_FLOW.value else None) + version_knowledge_ids = list(self.knowledge_id_list) + version_knowledge_ids.extend(flow_ids) + if self.knowledge_version_map is None: + validate_selection(selection, application.workspace_id, version_knowledge_ids) + self.knowledge_version_map = pin_selection(selection, application.workspace_id, self.chat_id) + elif self.knowledge_version_map: + pin_selection({}, application.workspace_id, self.chat_id, self.knowledge_version_map) return application def get_chat_user(self, asker=None): @@ -155,6 +166,7 @@ class ChatInfo: model_params_setting = credential.get_model_params_setting_form(model.model_name).get_default_form_data() return { 'knowledge_id_list': self.knowledge_id_list, + 'knowledge_version_map': self.knowledge_version_map or {}, 'exclude_document_id_list': self.exclude_document_id_list, 'exclude_paragraph_id_list': [], 'top_n': 3 if knowledge_setting.get('top_n') is None else knowledge_setting.get('top_n'), @@ -204,6 +216,7 @@ class ChatInfo: if not self.debug: if not QuerySet(Chat).filter(id=self.chat_id).exists(): Chat(id=self.chat_id, application_id=self.application_id, abstract=question[0:1024], + meta={'knowledge_versions': self.knowledge_version_map or {}}, chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type, asker=self.get_chat_user()).save() @@ -244,6 +257,7 @@ class ChatInfo: if not self.debug: if not QuerySet(Chat).filter(id=self.chat_id).exists(): Chat(id=self.chat_id, application_id=self.application_id, abstract=chat_record.problem_text[0:1024], + meta={'knowledge_versions': self.knowledge_version_map or {}}, chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type, asker=self.get_chat_user()).save() else: @@ -260,6 +274,7 @@ class ChatInfo: 'chat_user_type': self.chat_user_type, 'knowledge_id_list': self.knowledge_id_list, 'exclude_document_id_list': self.exclude_document_id_list, + 'knowledge_version_map': self.knowledge_version_map, 'application_id': self.application_id, 'chat_record_list': [self.chat_record_to_map(c) for c in self.chat_record_list][-20:], 'debug': self.debug @@ -309,6 +324,7 @@ class ChatInfo: chat_info_dict.get('application_id'), debug=chat_info_dict.get('debug')) c.chat_record_list = [ChatInfo.map_to_chat_record(c_r) for c_r in chat_info_dict.get('chat_record_list')] + c.knowledge_version_map = chat_info_dict.get('knowledge_version_map') return c @staticmethod diff --git a/apps/chat/serializers/chat.py b/apps/chat/serializers/chat.py index c05a517dc..4296a45c2 100644 --- a/apps/chat/serializers/chat.py +++ b/apps/chat/serializers/chat.py @@ -406,7 +406,8 @@ class ChatSerializers(serializers.Serializer): 'workspace_id': workspace_id, 'debug': debug, 'chat_user': chat_info.get_chat_user(), - 'application_id': str(chat_info.application_id)}, + 'application_id': str(chat_info.application_id), + 'knowledge_version_map': chat_info.knowledge_version_map or {}}, WorkFlowPostHandler(chat_info), base_to_response, form_data, image_list, document_list, audio_list, video_list, @@ -483,6 +484,9 @@ class ChatSerializers(serializers.Serializer): is_active=False)] chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), knowledge_id_list, exclude_document_id_list, application.id) + previous = Chat.objects.filter(pk=chat_id).first() + if previous and 'knowledge_versions' in (previous.meta or {}): + chat_info.knowledge_version_map = previous.meta['knowledge_versions'] chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5]) chat_record_list.sort(key=lambda r: r.create_time) for chat_record in chat_record_list: @@ -492,6 +496,9 @@ class ChatSerializers(serializers.Serializer): def re_open_chat_work_flow(self, chat_id, application): chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), [], [], application.id) + previous = Chat.objects.filter(pk=chat_id).first() + if previous and 'knowledge_versions' in (previous.meta or {}): + chat_info.knowledge_version_map = previous.meta['knowledge_versions'] chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5]) chat_record_list.sort(key=lambda r: r.create_time) for chat_record in chat_record_list: diff --git a/apps/common/auth/authenticate.py b/apps/common/auth/authenticate.py index a282f7aa6..567c3f0ac 100644 --- a/apps/common/auth/authenticate.py +++ b/apps/common/auth/authenticate.py @@ -86,6 +86,8 @@ class TokenAuth(TokenAuthentication): for handle in handles: if handle.support(request, token, token_details.get_token_details): authenticated = handle.handle(request, token, token_details.get_token_details) + from knowledge.versions import guard_snapshot_request + guard_snapshot_request(request) from common.constants.permission_constants import ChatAuth if isinstance(authenticated[1], ChatAuth): from system_manage.access_control import enforce diff --git a/apps/folders/serializers/folder.py b/apps/folders/serializers/folder.py index bca0772a2..b63713968 100644 --- a/apps/folders/serializers/folder.py +++ b/apps/folders/serializers/folder.py @@ -245,7 +245,10 @@ class FolderSerializer(serializers.Serializer): nodes = Folder.objects.filter(id=self.data.get('id')).get_descendants(include_self=True) for node in nodes: # 删除相关的资源 - source_ids = (Source.objects.filter(folder_id=node.id) + source_query = Source.objects.filter(folder_id=node.id) + if Source is Knowledge: + source_query = source_query.exclude(meta__has_key='_version_snapshot') + source_ids = (source_query .annotate(id_str=Cast('id', TextField())) .values_list('id_str', flat=True)) # 检查文件夹是否存在未授权当前用户的资源 @@ -263,7 +266,10 @@ class FolderSerializer(serializers.Serializer): def delete_source(self, node): Source = get_source_type(self.data.get('source')) # noqa - source_ids = Source.objects.filter(folder_id=node.id).values_list('id', flat=True) + source_query = Source.objects.filter(folder_id=node.id) + if Source is Knowledge: + source_query = source_query.exclude(meta__has_key='_version_snapshot') + source_ids = source_query.values_list('id', flat=True) source = self.data.get('source') for source_id in source_ids: diff --git a/apps/knowledge/models/__init__.py b/apps/knowledge/models/__init__.py index 4a43fce73..aee950f60 100644 --- a/apps/knowledge/models/__init__.py +++ b/apps/knowledge/models/__init__.py @@ -1 +1,2 @@ from .knowledge import * +from .version import KnowledgeVersion, KnowledgeVersionLease diff --git a/apps/knowledge/models/knowledge.py b/apps/knowledge/models/knowledge.py index 3b1a43637..2fd5e6987 100644 --- a/apps/knowledge/models/knowledge.py +++ b/apps/knowledge/models/knowledge.py @@ -92,7 +92,7 @@ class KnowledgeRebuildTask(AppModelMixin): class Meta: db_table = "knowledge_rebuild_task" indexes = [ - models.Index(fields=["knowledge", "status"]), + models.Index(fields=["knowledge", "status"], name='knowledge_r_knowled_0a6e2f_idx'), ] @@ -170,6 +170,7 @@ class Knowledge(AppModelMixin): file_size_limit = models.IntegerField(verbose_name="文件大小限制", default=100) file_count_limit = models.IntegerField(verbose_name="文件数量限制", default=50) meta = models.JSONField(verbose_name="元数据", default=dict) + version_sequence = models.PositiveIntegerField(default=0, editable=False) class Meta: db_table = "knowledge" diff --git a/apps/knowledge/serializers/knowledge.py b/apps/knowledge/serializers/knowledge.py index 6725d2d6f..62f294a5f 100644 --- a/apps/knowledge/serializers/knowledge.py +++ b/apps/knowledge/serializers/knowledge.py @@ -167,7 +167,8 @@ class KnowledgeSerializer(serializers.Serializer): query_set_dict['knowledge_custom_sql'] = QuerySet(model=get_dynamics_model({ 'knowledge.workspace_id': models.CharField(), - })).filter(**{'knowledge.workspace_id': workspace_id}) + 'knowledge.meta': models.JSONField(), + })).filter(**{'knowledge.workspace_id': workspace_id}).exclude(**{'knowledge.meta__has_key': '_version_snapshot'}) # query_set_dict['folder_query_set'] = folder_query_set if not workspace_manage: query_set_dict['workspace_user_resource_permission_query_set'] = QuerySet( @@ -440,6 +441,9 @@ class KnowledgeSerializer(serializers.Serializer): def delete(self): self.is_valid() knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id")) + from knowledge.models import KnowledgeVersion + if KnowledgeVersion.objects.filter(knowledge=knowledge).exists(): + raise AppApiException(500, '请先删除该知识库的历史版本') QuerySet(Document).filter(knowledge=knowledge).delete() QuerySet(ProblemParagraphMapping).filter(knowledge=knowledge).delete() QuerySet(Paragraph).filter(knowledge=knowledge).delete() diff --git a/apps/knowledge/urls.py b/apps/knowledge/urls.py index dbde32ae3..a6aff55b7 100644 --- a/apps/knowledge/urls.py +++ b/apps/knowledge/urls.py @@ -1,10 +1,13 @@ from django.urls import path from . import views +from .views.version import KnowledgeVersionView app_name = "knowledge" # @formatter:off urlpatterns = [ + path('workspace//knowledge//versions', KnowledgeVersionView.as_view()), + path('workspace//knowledge//versions/', KnowledgeVersionView.as_view()), path('workspace/knowledge/document/template/export', views.Template.as_view()), path('workspace/knowledge/document/table_template/export', views.TableTemplate.as_view()), path('workspace//knowledge', views.KnowledgeView.as_view()), diff --git a/apps/system_manage/platform_access.py b/apps/system_manage/platform_access.py index a20c7c4f9..e733b8256 100644 --- a/apps/system_manage/platform_access.py +++ b/apps/system_manage/platform_access.py @@ -214,7 +214,10 @@ def catalog(): resources = [] for kind, model in resource_models().items(): fields = ['id', 'name'] + (['provider', 'model_type'] if kind == 'MODEL' else []) - for item in model.objects.filter(workspace_id='default').values(*fields).order_by('name'): + query = model.objects.filter(workspace_id='default') + if kind == 'KNOWLEDGE': + query = query.exclude(meta__has_key='_version_snapshot') + for item in query.values(*fields).order_by('name'): resources.append({**item, 'id': str(item['id']), 'resource_type': kind}) return resources @@ -230,6 +233,8 @@ def validate_target(resource_type, target): model = resource_models().get(resource_type) try: valid = model is not None and model.objects.filter(pk=target, workspace_id='default').exists() + if valid and resource_type == 'KNOWLEDGE': + valid = not model.objects.filter(pk=target, meta__has_key='_version_snapshot').exists() except (ValueError, TypeError, DjangoValidationError): valid = False if not valid: diff --git a/apps/system_manage/sql/check_member_permission_target_exists.sql b/apps/system_manage/sql/check_member_permission_target_exists.sql index dc21460f5..bd64370e8 100644 --- a/apps/system_manage/sql/check_member_permission_target_exists.sql +++ b/apps/system_manage/sql/check_member_permission_target_exists.sql @@ -7,7 +7,7 @@ FROM (SELECT * auth_target_type FROM (SELECT "id"::text, 'KNOWLEDGE' AS "auth_target_type" - FROM knowledge + FROM (SELECT * FROM knowledge WHERE NOT (meta ? '_version_snapshot')) knowledge WHERE workspace_id = %s UNION SELECT "id"::text, diff --git a/apps/system_manage/sql/get_knowledge_user_resource_permission.sql b/apps/system_manage/sql/get_knowledge_user_resource_permission.sql index 8245c43fa..422a79fe8 100644 --- a/apps/system_manage/sql/get_knowledge_user_resource_permission.sql +++ b/apps/system_manage/sql/get_knowledge_user_resource_permission.sql @@ -14,7 +14,7 @@ FROM ( "type"::varchar AS "icon", folder_id, create_time - FROM knowledge + FROM (SELECT * FROM knowledge WHERE NOT (meta ? '_version_snapshot')) knowledge ${query_set} UNION SELECT knowledge_folder."id"::text, diff --git a/apps/system_manage/sql/get_user_resource_permission.sql b/apps/system_manage/sql/get_user_resource_permission.sql index 00ccb30e0..c8fe62bf7 100644 --- a/apps/system_manage/sql/get_user_resource_permission.sql +++ b/apps/system_manage/sql/get_user_resource_permission.sql @@ -8,7 +8,7 @@ FROM (SELECT "id", workspace_id, "type"::varchar AS "icon", folder_id - FROM knowledge + FROM (SELECT * FROM knowledge WHERE NOT (meta ? '_version_snapshot')) knowledge ${knowledge_query_set} UNION SELECT "id", diff --git a/ui/src/views/application/ApplicationSetting.vue b/ui/src/views/application/ApplicationSetting.vue index 7dc67f8cf..89fe0bb9a 100644 --- a/ui/src/views/application/ApplicationSetting.vue +++ b/ui/src/views/application/ApplicationSetting.vue @@ -247,6 +247,13 @@ + @@ -326,10 +333,9 @@