From 91f9199020e8258e02515cfe0515fcba1afdb28c Mon Sep 17 00:00:00 2001 From: panyy Date: Fri, 6 Nov 2026 14:51:58 +0800 Subject: [PATCH] chore(release): include refact runtime sources and frontend lockfile --- .gitignore | 8 +- .../flow/step_node/component_node/__init__.py | 1 + .../component_node/base_component_node.py | 64 ++ apps/knowledge/embedding_guard.py | 49 + .../migrations/0006_knowledge_versions.py | 82 ++ apps/knowledge/models/version.py | 36 + apps/knowledge/test_versions.py | 350 +++++++ apps/knowledge/versions.py | 331 +++++++ apps/knowledge/views/version.py | 55 ++ apps/models_provider/context_policy.py | 34 + apps/models_provider/test_context_policy.py | 74 ++ apps/system_manage/ai_hub_service_api.py | 104 +++ apps/system_manage/artifact_settings.py | 26 + apps/system_manage/heading_service.py | 70 ++ apps/system_manage/heading_structure.py | 456 +++++++++ apps/system_manage/mcp_agent.py | 113 +++ apps/system_manage/mcp_business.py | 388 ++++++++ apps/system_manage/mcp_registry.py | 126 +++ apps/system_manage/mcp_runtime.py | 180 ++++ apps/system_manage/mcp_scheduler.py | 233 +++++ apps/system_manage/mcp_workflows.py | 224 +++++ apps/system_manage/parsers/native_outline.py | 153 ++++ apps/system_manage/summary_plan.py | 169 ++++ apps/system_manage/summary_sections.py | 447 +++++++++ apps/system_manage/test_ai_hub_progress.py | 24 + apps/system_manage/test_ai_hub_service_api.py | 146 +++ apps/system_manage/test_heading_structure.py | 152 +++ .../test_heading_structure_api.py | 131 +++ apps/system_manage/test_mcp_business.py | 433 +++++++++ apps/system_manage/test_mcp_engine.py | 348 +++++++ apps/system_manage/test_my_mindmaps.py | 187 ++++ apps/system_manage/test_summary_sections.py | 212 +++++ apps/system_manage/test_workbench_defaults.py | 83 ++ apps/system_manage/views/mcp_engine.py | 171 ++++ apps/system_manage/workbench_defaults.py | 108 +++ scripts/check_ai_hub_service_api.py | 52 ++ scripts/check_heading_structure.py | 52 ++ scripts/check_knowledge_versions.py | 62 ++ scripts/check_mcp_engine.py | 52 ++ scripts/check_my_mindmaps.py | 52 ++ scripts/check_native_parsers.py | 24 + ui/pnpm-lock.yaml | 866 ++++++++++++------ ui/pnpm-workspace.yaml | 5 + ui/src/api/knowledge/versions.ts | 13 + ui/src/api/system/mcp-engine.ts | 14 + .../components/engine-call-records/index.vue | 15 + .../components/engine-task-detail/index.vue | 43 + ui/src/components/system-pagination/index.vue | 16 + .../component/KnowledgeVersionSelect.vue | 42 + .../knowledge/component/KnowledgeVersions.vue | 239 +++++ .../components/HeadingStructurePanel.vue | 84 ++ .../workbench/components/MyMindmapsDialog.vue | 268 ++++++ .../workbench/utils/contextDefaults.ts | 9 + .../workbench/utils/exportPreparation.ts | 42 + ui/src/views/system/mcp-engine/index.vue | 113 +++ ui/src/views/system/mindmap/AdvancedFlows.vue | 29 + .../system/mindmap/AdvancedFlowsPage.vue | 12 + .../system/mindmap/WorkbenchSettings.vue | 56 ++ ui/src/workflow/icons/component-node-icon.vue | 5 + ui/src/workflow/nodes/component-node/index.ts | 6 + .../workflow/nodes/component-node/index.vue | 102 +++ 61 files changed, 7744 insertions(+), 297 deletions(-) create mode 100644 apps/application/flow/step_node/component_node/__init__.py create mode 100644 apps/application/flow/step_node/component_node/base_component_node.py create mode 100644 apps/knowledge/embedding_guard.py create mode 100644 apps/knowledge/migrations/0006_knowledge_versions.py create mode 100644 apps/knowledge/models/version.py create mode 100644 apps/knowledge/test_versions.py create mode 100644 apps/knowledge/versions.py create mode 100644 apps/knowledge/views/version.py create mode 100644 apps/models_provider/context_policy.py create mode 100644 apps/models_provider/test_context_policy.py create mode 100644 apps/system_manage/ai_hub_service_api.py create mode 100644 apps/system_manage/artifact_settings.py create mode 100644 apps/system_manage/heading_service.py create mode 100644 apps/system_manage/heading_structure.py create mode 100644 apps/system_manage/mcp_agent.py create mode 100644 apps/system_manage/mcp_business.py create mode 100644 apps/system_manage/mcp_registry.py create mode 100644 apps/system_manage/mcp_runtime.py create mode 100644 apps/system_manage/mcp_scheduler.py create mode 100644 apps/system_manage/mcp_workflows.py create mode 100644 apps/system_manage/parsers/native_outline.py create mode 100644 apps/system_manage/summary_plan.py create mode 100644 apps/system_manage/summary_sections.py create mode 100644 apps/system_manage/test_ai_hub_progress.py create mode 100644 apps/system_manage/test_ai_hub_service_api.py create mode 100644 apps/system_manage/test_heading_structure.py create mode 100644 apps/system_manage/test_heading_structure_api.py create mode 100644 apps/system_manage/test_mcp_business.py create mode 100644 apps/system_manage/test_mcp_engine.py create mode 100644 apps/system_manage/test_my_mindmaps.py create mode 100644 apps/system_manage/test_summary_sections.py create mode 100644 apps/system_manage/test_workbench_defaults.py create mode 100644 apps/system_manage/views/mcp_engine.py create mode 100644 apps/system_manage/workbench_defaults.py create mode 100644 scripts/check_ai_hub_service_api.py create mode 100644 scripts/check_heading_structure.py create mode 100644 scripts/check_knowledge_versions.py create mode 100644 scripts/check_mcp_engine.py create mode 100644 scripts/check_my_mindmaps.py create mode 100644 scripts/check_native_parsers.py create mode 100644 ui/pnpm-workspace.yaml create mode 100644 ui/src/api/knowledge/versions.ts create mode 100644 ui/src/api/system/mcp-engine.ts create mode 100644 ui/src/components/engine-call-records/index.vue create mode 100644 ui/src/components/engine-task-detail/index.vue create mode 100644 ui/src/components/system-pagination/index.vue create mode 100644 ui/src/views/knowledge/component/KnowledgeVersionSelect.vue create mode 100644 ui/src/views/knowledge/component/KnowledgeVersions.vue create mode 100644 ui/src/views/mindmap/workbench/components/HeadingStructurePanel.vue create mode 100644 ui/src/views/mindmap/workbench/components/MyMindmapsDialog.vue create mode 100644 ui/src/views/mindmap/workbench/utils/contextDefaults.ts create mode 100644 ui/src/views/mindmap/workbench/utils/exportPreparation.ts create mode 100644 ui/src/views/system/mcp-engine/index.vue create mode 100644 ui/src/views/system/mindmap/AdvancedFlows.vue create mode 100644 ui/src/views/system/mindmap/AdvancedFlowsPage.vue create mode 100644 ui/src/views/system/mindmap/WorkbenchSettings.vue create mode 100644 ui/src/workflow/icons/component-node-icon.vue create mode 100644 ui/src/workflow/nodes/component-node/index.ts create mode 100644 ui/src/workflow/nodes/component-node/index.vue diff --git a/.gitignore b/.gitignore index 17f102d33..02d23e33d 100644 --- a/.gitignore +++ b/.gitignore @@ -189,4 +189,10 @@ apps/models_provider/impl/tencent_model_provider/credential/stt.py apps/models_provider/impl/tencent_model_provider/model/stt.py tmp/ config.yml -.SANDBOX_BANNED_HOSTS \ No newline at end of file +.SANDBOX_BANNED_HOSTS + +# Local verification outputs; release sources remain visible to Git. +/ui/dist*/ +/ui/.summary-build/ +/jieba.cache +/mcp-engine-*/ diff --git a/apps/application/flow/step_node/component_node/__init__.py b/apps/application/flow/step_node/component_node/__init__.py new file mode 100644 index 000000000..1df7af511 --- /dev/null +++ b/apps/application/flow/step_node/component_node/__init__.py @@ -0,0 +1 @@ +from .base_component_node import BaseComponentNode diff --git a/apps/application/flow/step_node/component_node/base_component_node.py b/apps/application/flow/step_node/component_node/base_component_node.py new file mode 100644 index 000000000..51e715da9 --- /dev/null +++ b/apps/application/flow/step_node/component_node/base_component_node.py @@ -0,0 +1,64 @@ +from copy import deepcopy +from rest_framework import serializers +from rest_framework.exceptions import ValidationError +from application.flow.i_step_node import INode, NodeResult +from system_manage import mcp_runtime + + +class ComponentParams(serializers.Serializer): + component_id = serializers.CharField() + parameters = serializers.DictField(default=dict) + sources = serializers.DictField(default=dict) + timeout = serializers.IntegerField(default=60, min_value=1, max_value=600) + retries = serializers.IntegerField(default=0, min_value=0, max_value=2) + priority = serializers.IntegerField(default=5, min_value=0, max_value=9) + + +def workflow_identity(workflow): + from application.models import Application + from users.models import User + body = workflow.get_body() + app = Application.objects.filter(id=body['application_id'], workspace_id=body['workspace_id']).first() + if not app or not app.user_id: + raise ValidationError('应用不存在或没有可用的所有者') + user = User.objects.get(id=body.get('engine_user_id') or app.user_id, is_active=True) + return body, user + + +class BaseComponentNode(INode): + type = 'component-node' + + def get_node_params_serializer_class(self): + return ComponentParams + + def _run(self): + return self.execute(**self.node_params_serializer.data) + + def execute(self, component_id, parameters=None, sources=None, timeout=60, retries=0, priority=5, **kwargs): + body, user = workflow_identity(self.workflow_manage) + def render(value): + if isinstance(value, str): return self.workflow_manage.generate_prompt(value) + if isinstance(value, dict): return {key: render(item) for key, item in value.items()} + if isinstance(value, list): return [render(item) for item in value] + return value + params = deepcopy(parameters or {}) + for key, value in params.items(): + if (sources or {}).get(key) == 'reference': + if not isinstance(value, list) or len(value) < 2: + raise ValidationError('请选择有效的上游变量') + params[key] = self.workflow_manage.get_reference_field(value[0], value[1:]) + else: + params[key] = render(value) + result = mcp_runtime.invoke(body['workspace_id'], user, component_id, params, + parent=body.get('engine_task_id'), application_id=body['application_id'], chat_id=body.get('chat_id'), + scope={**body, 'node_id': self.id, 'source': 'workflow-component'}, + timeout=timeout, retries=retries, priority=priority) + return NodeResult(result, {}) + + def save_context(self, details, workflow_manage): + self.context.update({key: details.get(key) for key in ('text', 'structured', 'result', 'task_id', 'component_id')}) + + def get_details(self, index, **kwargs): + return {'name': self.node.properties.get('stepName'), 'index': index, 'type': self.type, + 'status': self.status, 'err_message': self.err_message, 'run_time': self.context.get('run_time'), + **{key: self.context.get(key) for key in ('component_id', 'text', 'structured', 'task_id')}} diff --git a/apps/knowledge/embedding_guard.py b/apps/knowledge/embedding_guard.py new file mode 100644 index 000000000..8ec931ab4 --- /dev/null +++ b/apps/knowledge/embedding_guard.py @@ -0,0 +1,49 @@ +"""Guard changes that would invalidate linked assistants or version snapshots.""" +from django.db.models import OuterRef, Subquery + +from application.models import Application, ApplicationKnowledgeMapping, ApplicationVersion +from knowledge.models import KnowledgeVersion + + +VERSION_REASON = '知识库已有历史版本,暂不支持更换嵌入模型或影响向量结果的参数。' +ASSISTANT_REASON = '关联助手使用了多个知识库,暂不支持更换嵌入模型或影响向量结果的参数。' + + +def flow_knowledge_ids(value): + """Find static knowledge references, including nodes nested in loop bodies.""" + ids = set() + if isinstance(value, dict): + if value.get('type') == 'search-knowledge-node': + data = (value.get('properties') or {}).get('node_data') or {} + ids.update(str(item) for item in (data.get('knowledge_id_list') or []) if item) + for child in value.values(): + if isinstance(child, (dict, list)): + ids.update(flow_knowledge_ids(child)) + elif isinstance(value, list): + for child in value: + if isinstance(child, (dict, list)): + ids.update(flow_knowledge_ids(child)) + return ids + + +def embedding_change_reason(knowledge): + if KnowledgeVersion.objects.filter(knowledge_id=knowledge.id).exists(): + return VERSION_REASON + target = str(knowledge.id) + mappings = {} + for app_id, knowledge_id in ApplicationKnowledgeMapping.objects.values_list('application_id', 'knowledge_id'): + mappings.setdefault(str(app_id), set()).add(str(knowledge_id)) + for app in Application.objects.only('id', 'type', 'work_flow').iterator(): + ids = mappings.get(str(app.id), set()) | flow_knowledge_ids(app.work_flow) + if target in ids and len(ids) > 1: + return ASSISTANT_REASON + latest = ApplicationVersion.objects.filter(application_id=OuterRef('application_id')).order_by( + '-create_time', '-id').values('id')[:1] + for version in ApplicationVersion.objects.filter(id=Subquery(latest)).only( + 'application_id', 'knowledge_setting', 'work_flow').iterator(): + setting = version.knowledge_setting or {} + ids = {str(item) for item in setting.get('_knowledge_ids', mappings.get(str(version.application_id), set()))} + ids |= flow_knowledge_ids(version.work_flow) + if target in ids and len(ids) > 1: + return ASSISTANT_REASON + return '' diff --git a/apps/knowledge/migrations/0006_knowledge_versions.py b/apps/knowledge/migrations/0006_knowledge_versions.py new file mode 100644 index 000000000..69b1a22cc --- /dev/null +++ b/apps/knowledge/migrations/0006_knowledge_versions.py @@ -0,0 +1,82 @@ +# Generated by Django 5.2.8 on 2026-10-27 03:14 + +import django.db.models.deletion +import uuid_utils.compat +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('knowledge', '0005_rebuild_task_attempt'), + ('users', '0002_user_activation_and_role_assignment'), + ] + + operations = [ + migrations.AddField( + model_name='knowledge', + name='version_sequence', + field=models.PositiveIntegerField(default=0, editable=False), + ), + migrations.CreateModel( + name='KnowledgeVersion', + fields=[ + ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')), + ('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')), + ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False)), + ('number', models.PositiveIntegerField()), + ('name', models.CharField(max_length=128)), + ('description', models.CharField(blank=True, default='', max_length=512)), + ('creator_name', models.CharField(default='', max_length=128)), + ('document_count', models.PositiveIntegerField(default=0)), + ('paragraph_count', models.PositiveIntegerField(default=0)), + ('vector_count', models.PositiveIntegerField(default=0)), + ('search_ready', models.BooleanField(default=False)), + ('content_hash', models.CharField(max_length=64)), + ('config', models.JSONField(default=dict)), + ], + options={ + 'db_table': 'knowledge_version', + 'ordering': ['-number'], + }, + ), + migrations.CreateModel( + name='KnowledgeVersionLease', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chat_id', models.CharField(max_length=128)), + ('expires_at', models.DateTimeField(db_index=True)), + ], + options={ + 'db_table': 'knowledge_version_lease', + }, + ), + migrations.AddField( + model_name='knowledgeversion', + name='creator', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='users.user'), + ), + migrations.AddField( + model_name='knowledgeversion', + name='knowledge', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='versions', to='knowledge.knowledge'), + ), + migrations.AddField( + model_name='knowledgeversion', + name='snapshot', + field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='knowledge.knowledge'), + ), + migrations.AddField( + model_name='knowledgeversionlease', + name='version', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='leases', to='knowledge.knowledgeversion'), + ), + migrations.AddConstraint( + model_name='knowledgeversion', + constraint=models.UniqueConstraint(fields=('knowledge', 'number'), name='knowledge_version_number_unique'), + ), + migrations.AddConstraint( + model_name='knowledgeversionlease', + constraint=models.UniqueConstraint(fields=('version', 'chat_id'), name='knowledge_version_chat_unique'), + ), + ] diff --git a/apps/knowledge/models/version.py b/apps/knowledge/models/version.py new file mode 100644 index 000000000..7e8248c15 --- /dev/null +++ b/apps/knowledge/models/version.py @@ -0,0 +1,36 @@ +"""Immutable knowledge releases and short-lived conversation references.""" +import uuid_utils.compat as uuid +from django.db import models +from common.mixins.app_model_mixin import AppModelMixin + + +class KnowledgeVersion(AppModelMixin): + id = models.UUIDField(primary_key=True, default=uuid.uuid7, editable=False) + knowledge = models.ForeignKey('knowledge.Knowledge', on_delete=models.PROTECT, related_name='versions') + snapshot = models.OneToOneField('knowledge.Knowledge', on_delete=models.PROTECT, related_name='+') + number = models.PositiveIntegerField() + name = models.CharField(max_length=128) + description = models.CharField(max_length=512, default='', blank=True) + creator = models.ForeignKey('users.User', null=True, on_delete=models.SET_NULL) + creator_name = models.CharField(max_length=128, default='') + document_count = models.PositiveIntegerField(default=0) + paragraph_count = models.PositiveIntegerField(default=0) + vector_count = models.PositiveIntegerField(default=0) + search_ready = models.BooleanField(default=False) + content_hash = models.CharField(max_length=64) + config = models.JSONField(default=dict) + + class Meta: + db_table = 'knowledge_version' + ordering = ['-number'] + constraints = [models.UniqueConstraint(fields=['knowledge', 'number'], name='knowledge_version_number_unique')] + + +class KnowledgeVersionLease(models.Model): + version = models.ForeignKey(KnowledgeVersion, on_delete=models.CASCADE, related_name='leases') + chat_id = models.CharField(max_length=128) + expires_at = models.DateTimeField(db_index=True) + + class Meta: + db_table = 'knowledge_version_lease' + constraints = [models.UniqueConstraint(fields=['version', 'chat_id'], name='knowledge_version_chat_unique')] diff --git a/apps/knowledge/test_versions.py b/apps/knowledge/test_versions.py new file mode 100644 index 000000000..3dc7e1d57 --- /dev/null +++ b/apps/knowledge/test_versions.py @@ -0,0 +1,350 @@ +import uuid +import json +from datetime import timedelta +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import patch +from django.db import transaction +from django.test import RequestFactory +from django.utils import timezone +from rest_framework.exceptions import ValidationError, NotFound +from common.exception.app_exception import AppUnauthorizedFailed, NotFound404 +from knowledge import versions as v +from knowledge.models import (KnowledgeFolder, Knowledge, Document, Paragraph, Problem, ProblemParagraphMapping, + Tag, DocumentTag, Embedding, File, KnowledgeVersion, KnowledgeVersionLease) +from users.models import User +from models_provider.models import Model +from application.models import Application, ApplicationFolder, ApplicationVersion, ApplicationKnowledgeMapping + + +class VersionTests(TestCase): + def setUp(self): + self.atomic = transaction.atomic(); self.atomic.__enter__() + from django.db import connection + if connection.vendor == 'postgresql': + with connection.cursor() as cursor: + cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ') + self.user = User.objects.create(username='CODEX_VERSION_TEST', nick_name='版本测试', role='ADMIN') + self.folder = KnowledgeFolder.objects.create(id='default', name='默认') + ApplicationFolder.objects.create(id='default', name='默认') + self.model = Model.objects.create(name='测试向量', model_name='test-embedding', provider='test', + model_type='EMBEDDING', credential='encrypted-test', user=self.user) + self.root = Knowledge.objects.create(name='CODEX知识库', desc='原始', folder=self.folder, user=self.user, + embedding_model=self.model, meta={'embedding_params': {'batch_size': 8}, 'segmentation': {'chunk_size': 500}}) + self.doc = Document.objects.create(knowledge=self.root, name='样例', char_length=8, status='nn2') + self.para = Paragraph.objects.create(knowledge=self.root, document=self.doc, content='旧答案:100元', position=1) + self.problem = Problem.objects.create(knowledge=self.root, content='价格多少') + ProblemParagraphMapping.objects.create(knowledge=self.root, document=self.doc, paragraph=self.para, problem=self.problem) + tag = Tag.objects.create(knowledge=self.root, key='部门', value='测试') + DocumentTag.objects.create(document=self.doc, tag=tag) + Embedding.objects.create(id=str(uuid.uuid4()), knowledge=self.root, document=self.doc, paragraph=self.para, + source_id=str(self.para.id), source_type='1', embedding='[1,0,0]', search_vector='price') + + def tearDown(self): + transaction.set_rollback(True); self.atomic.__exit__(None, None, None) + + def create(self, name='初版'): + return v.create_version('default', self.root.id, self.user, {'name': name}) + + def edit_embedding(self, values): + from knowledge.serializers.knowledge import KnowledgeSerializer + return KnowledgeSerializer.Operate(data={'user_id': str(self.user.id), 'workspace_id': 'default', + 'knowledge_id': str(self.root.id)}).edit(values, select_one=False) + + def test_embedding_guard_blocks_version_but_allows_unchanged_and_batch(self): + from common.exception.app_exception import AppApiException + self.create() + self.edit_embedding({'embedding_model_id': str(self.model.id), 'embedding_params': {'batch_size': 12}}) + with self.assertRaises(AppApiException): + self.edit_embedding({'embedding_model_id': str(uuid.uuid4())}) + with self.assertRaises(AppApiException): + self.edit_embedding({'embedding_params': {'batch_size': 12, 'normalize': False}}) + + def test_embedding_guard_detects_current_and_published_assistants(self): + from common.exception.app_exception import AppApiException + other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model) + app = Application.objects.create(name='助手', user=self.user) + ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root) + self.edit_embedding({'embedding_model_id': str(self.model.id)}) + ApplicationKnowledgeMapping.objects.create(application=app, knowledge=other) + with self.assertRaises(AppApiException): + self.edit_embedding({'embedding_model_id': str(uuid.uuid4())}) + ApplicationVersion.objects.create(application=app, knowledge_setting={ + '_knowledge_ids': [str(self.root.id), str(other.id)]}) + ApplicationKnowledgeMapping.objects.filter(application=app, knowledge=other).delete() + with self.assertRaises(AppApiException): + self.edit_embedding({'embedding_model_id': str(uuid.uuid4())}) + ApplicationVersion.objects.create(application=app, knowledge_setting={'_knowledge_ids': [str(self.root.id)]}) + self.edit_embedding({'embedding_model_id': str(self.model.id)}) + + def test_embedding_guard_detects_workflow_and_allows_single_reference(self): + from knowledge.embedding_guard import embedding_change_reason, ASSISTANT_REASON + other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model) + node = {'type': 'search-knowledge-node', 'properties': {'node_data': { + 'knowledge_id_list': [str(self.root.id)]}}} + app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user, + work_flow={'nodes': [node]}) + self.assertEqual(embedding_change_reason(self.root), '') + node['properties']['node_data']['knowledge_id_list'].append(str(other.id)) + app.work_flow = {'nodes': [node]}; app.save() + self.assertEqual(embedding_change_reason(self.root), ASSISTANT_REASON) + + def test_embedding_guard_allows_model_change_with_single_assistant(self): + other_model = Model.objects.create(name='备用向量', model_name='test-embedding-2', provider='test', + model_type='EMBEDDING', credential='encrypted-test', user=self.user) + app = Application.objects.create(name='助手', user=self.user) + ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root) + self.edit_embedding({'embedding_model_id': str(other_model.id)}) + self.root.refresh_from_db() + self.assertEqual(self.root.embedding_model_id, other_model.id) + + def test_full_snapshot_and_id_remapping(self): + version = self.create() + self.assertTrue(version.search_ready) + self.assertEqual(version.document_count, 1) + paragraph = Paragraph.objects.get(knowledge=version.snapshot) + self.assertNotEqual(paragraph.id, self.para.id) + self.assertEqual(paragraph.content, self.para.content) + mapping = ProblemParagraphMapping.objects.get(knowledge=version.snapshot) + self.assertEqual(mapping.paragraph_id, paragraph.id) + self.assertEqual(mapping.document_id, paragraph.document_id) + self.assertEqual(DocumentTag.objects.filter(document__knowledge=version.snapshot).count(), 1) + vector = Embedding.objects.get(knowledge=version.snapshot) + self.assertEqual(vector.paragraph_id, paragraph.id) + self.assertEqual(vector.source_id, str(paragraph.id)) + + def test_live_edits_do_not_change_snapshot(self): + version = self.create() + self.para.content = '新答案:200元'; self.para.save() + self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, '旧答案:100元') + second = self.create('第二版') + self.assertEqual(Paragraph.objects.get(knowledge=second.snapshot).content, '新答案:200元') + self.assertNotEqual(version.content_hash, second.content_hash) + + def test_text_uuid_is_preserved(self): + self.para.content = '业务标识:' + str(self.doc.id); self.para.save() + version = self.create() + self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, self.para.content) + + def test_binary_files_shared_and_markdown_rewritten(self): + file = File(id=uuid.uuid4(), file_name='测试.png', loid=12345, source_id=str(self.root.id), + source_type='KNOWLEDGE', sha256_hash='test', file_size=10, meta={'knowledge_id': str(self.root.id)}) + File.objects.bulk_create([file]) + self.para.content = '![](./oss/file/' + str(file.id) + ')'; self.para.save() + version = self.create() + cloned = File.objects.get(source_id=str(version.snapshot_id)) + self.assertEqual(cloned.loid, file.loid) + self.assertNotEqual(cloned.id, file.id) + self.assertIn(str(cloned.id), Paragraph.objects.get(knowledge=version.snapshot).content) + with patch('knowledge.models.knowledge.select_one') as unlink: + v.delete_version('default', self.root.id, version.id) + unlink.assert_not_called() + self.assertTrue(File.objects.filter(pk=file.id).exists()) + + def test_version_delete_cleans_only_snapshot(self): + version = self.create(); snapshot_id = version.snapshot_id + v.delete_version('default', self.root.id, version.id) + for model in (Document, Paragraph, Problem, Embedding, Tag, ProblemParagraphMapping): + self.assertFalse(model.objects.filter(knowledge_id=snapshot_id).exists()) + self.assertFalse(Knowledge.objects.filter(pk=snapshot_id).exists()) + self.assertTrue(Paragraph.objects.filter(pk=self.para.id).exists()) + + def test_version_number_not_reused(self): + first = self.create(); v.delete_version('default', self.root.id, first.id) + self.assertEqual(self.create().number, 2) + + def test_create_transaction_rolls_back_on_index_failure(self): + with patch.object(v, '_build_index', side_effect=RuntimeError('index failed')), self.assertRaises(RuntimeError): self.create() + self.assertEqual(Knowledge.objects.count(), 1) + self.assertEqual(KnowledgeVersion.objects.count(), 0) + + def test_pending_processing_rejected(self): + self.doc.status = 'nn0'; self.doc.save() + with self.assertRaises(ValidationError): self.create() + + def test_missing_vectors_are_browsable_but_not_selectable(self): + Embedding.objects.all().delete(); version = self.create() + self.assertFalse(version.search_ready) + with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [self.root.id]) + + def test_cross_knowledge_selection_rejected(self): + version = self.create() + with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', []) + + def test_invalid_version_rejected(self): + with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): 'invalid'}, 'default', [self.root.id]) + + def test_cross_workspace_rejected(self): + version = self.create() + with self.assertRaises(NotFound404): v.get_version('other', self.root.id, version.id) + + def test_fixed_selection_and_live_lookup(self): + version = self.create() + self.assertEqual(v.resolve_search([self.root.id], {}, 'default'), [str(self.root.id)]) + self.assertEqual(v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default'), [str(version.snapshot_id)]) + + def test_document_scope_maps_to_frozen_documents(self): + version = self.create(); document = Document.objects.get(knowledge=version.snapshot) + self.assertEqual(v.resolve_document_scope([self.doc.id], {str(self.root.id): str(version.id)}, 'default'), [str(document.id)]) + self.assertEqual(v.resolve_document_scope([self.doc.id], {}, 'default'), [str(self.doc.id)]) + data = v.version_data(version) + self.assertNotIn('_document_ids', data['config']) + self.assertNotIn('model_identity', data['config']) + + def test_same_chat_keeps_latest_choice(self): + first = self.create() + selection = {str(self.root.id): 'latest'} + pinned = v.pin_selection(selection, 'default', 'chat-test') + self.create('新版') + self.assertEqual(v.pin_selection(selection, 'default', 'chat-test'), pinned) + self.assertEqual(pinned[str(self.root.id)], str(first.id)) + self.assertNotEqual(v.pin_selection(selection, 'default', 'new-chat'), pinned) + + def test_active_conversation_protects_delete(self): + version = self.create(); v.pin_selection({str(self.root.id): str(version.id)}, 'default', 'chat-test') + with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id) + KnowledgeVersionLease.objects.update(expires_at=timezone.now()-timedelta(seconds=1)) + v.delete_version('default', self.root.id, version.id) + + def test_assistant_pin_protects_delete(self): + version = self.create() + app = Application.objects.create(name='测试助手', user=self.user, + knowledge_setting={'version_selection': {str(self.root.id): str(version.id)}}) + with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id) + app.knowledge_setting = {}; app.save(); v.delete_version('default', self.root.id, version.id) + + def test_latest_assistant_protects_only_latest_ready_version(self): + first = self.create(); second = self.create('新版') + Application.objects.create(name='测试助手', user=self.user, + knowledge_setting={'version_selection': {str(self.root.id): 'latest'}}) + with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, second.id) + v.delete_version('default', self.root.id, first.id) + + def test_workflow_draft_and_published_version_protect_delete(self): + version = self.create() + flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': { + 'knowledge_id_list': [str(self.root.id)], + 'knowledge_setting': {'version_selection': {str(self.root.id): str(version.id)}}}}}]} + app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user, work_flow=flow) + with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id) + app.work_flow = {}; app.save() + published = ApplicationVersion.objects.create(application=app, type='WORK_FLOW', work_flow=flow) + with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id) + published.delete(); v.delete_version('default', self.root.id, version.id) + + def test_workflow_conflicting_versions_rejected(self): + flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': { + 'knowledge_setting': {'version_selection': {str(self.root.id): 'latest'}}}}}]} + with self.assertRaises(ValidationError): + v.application_selection({'version_selection': {str(self.root.id): 'different'}}, flow) + + def test_republication_releases_superseded_version(self): + version = self.create(); app = Application.objects.create(name='重新发布', user=self.user) + ApplicationVersion.objects.create(application=app, knowledge_setting={ + 'version_selection': {str(self.root.id): str(version.id)}}) + with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id) + ApplicationVersion.objects.create(application=app, knowledge_setting={}) + v.delete_version('default', self.root.id, version.id) + + def test_migration_target_cannot_be_snapshot(self): + version = self.create(); request = RequestFactory().post('/document/migrate') + request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(self.root.id), + 'target_knowledge_id': str(version.snapshot_id)}) + with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request) + + def test_published_chat_keeps_published_association(self): + from application.serializers.common import ChatInfo + version = self.create() + app = Application.objects.create(name='发布测试', user=self.user) + ApplicationVersion.objects.create(application=app, knowledge_setting={ + '_knowledge_ids': [str(self.root.id)], 'version_selection': {str(self.root.id): str(version.id)}}) + self.assertFalse(ApplicationKnowledgeMapping.objects.filter(application=app).exists()) + chat = ChatInfo(str(uuid.uuid4()), str(self.user.id), 'PLATFORM_USER', [], [], str(app.id)) + chat.get_application() + self.assertEqual(chat.knowledge_id_list, [str(self.root.id)]) + self.assertEqual(chat.knowledge_version_map, {str(self.root.id): str(version.id)}) + + def test_feedback_body_cannot_write_snapshot(self): + from application.serializers.application_chat_record import ApplicationChatRecordAddKnowledgeSerializer + version = self.create(); doc = Document.objects.get(knowledge=version.snapshot) + app = Application.objects.create(name='改进测试', user=self.user) + serializer = ApplicationChatRecordAddKnowledgeSerializer(data={'workspace_id': 'default', + 'application_id': str(app.id), 'knowledge_id': str(version.snapshot_id), + 'document_id': str(doc.id), 'chat_ids': []}) + with self.assertRaises(NotFound404): serializer.post_improve({}, with_valid=False) + + def test_model_changes_fail_explicitly(self): + version = self.create(); self.model.model_name = 'different'; self.model.save() + with self.assertRaises(ValidationError): v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default') + + def test_internal_snapshot_api_guard(self): + version = self.create(); request = RequestFactory().get('/admin/api/knowledge') + request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(version.snapshot_id)}) + with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request) + with self.assertRaises(NotFound404): v.root_knowledge('default', version.snapshot_id) + + def test_api_read_permission_and_workspace(self): + from rest_framework.test import APIRequestFactory, force_authenticate + from knowledge.views.version import KnowledgeVersionView + self.create(); factory = APIRequestFactory() + for roles, workspace, expected in [([], 'default', 403), (['ADMIN'], 'default', 200), (['ADMIN'], 'other', 404)]: + req = factory.get('/versions') + force_authenticate(req, user=self.user, token=SimpleNamespace(role_list=roles, permission_list=[])) + response = KnowledgeVersionView.as_view()(req, workspace_id=workspace, knowledge_id=self.root.id) + self.assertEqual(response.status_code, expected) + + def test_empty_name_rejected(self): + with self.assertRaises(ValidationError): self.create(' ') + + def test_real_postgres_retrieval_reads_frozen_content(self): + from django.db import connection + if connection.vendor != 'postgresql': + self.skipTest('Real pgvector retrieval requires isolated PostgreSQL schema') + from application.chat_pipeline.step.search_dataset_step.impl import base_search_dataset_step as search + version = self.create() + self.para.content = '新答案:200元'; self.para.save() + manage = SimpleNamespace(context={'knowledge_version_map': {str(self.root.id): str(version.id)}, 'chat_user_type': 'PLATFORM_USER'}) + model = SimpleNamespace(model_type='EMBEDDING', name='测试向量') + embedding = SimpleNamespace(embed_query=lambda text: [1.0, 0.0, 0.0]) + with patch.object(search, 'get_model_by_id', return_value=model), patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding): + result = search.BaseSearchDatasetStep().execute('价格', [str(self.root.id)], [], [], 3, 0.1, + search_mode='embedding', workspace_id='default', manage=manage) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].content, '旧答案:100元') + self.assertIn('v1', result[0].knowledge_name) + + def test_real_postgres_workflow_retrieves_frozen_document(self): + from django.db import connection + if connection.vendor != 'postgresql': self.skipTest('Real workflow pgvector retrieval requires PostgreSQL') + from application.flow.step_node.search_knowledge_node.impl import base_search_knowledge_node as search + version = self.create(); self.para.content = '新答案:200元'; self.para.save() + body = {'knowledge_version_map': {str(self.root.id): str(version.id)}, + 'chat_user_type': 'PLATFORM_USER', 'workspace_id': 'default'} + node = search.BaseSearchKnowledgeNode.__new__(search.BaseSearchKnowledgeNode) + node.context = {}; node.workflow_manage = SimpleNamespace(get_body=lambda: body, + get_reference_field=lambda *_: [str(self.doc.id)]) + embedding = SimpleNamespace(embed_query=lambda _: [1.0, 0.0, 0.0]) + with patch.object(search, 'get_model_instance_by_model_workspace_id'), \ + patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding), \ + patch.object(search.connection, 'close'): + result = node.execute([str(self.root.id)], {'top_n':3, 'similarity':0.1, 'search_mode':'embedding'}, + '价格', True, 'referencing', 'document', ['previous', 'documents']) + self.assertEqual(result.node_variable['paragraph_list'][0]['content'], '旧答案:100元') + + def test_api_create_browse_and_delete(self): + from rest_framework.test import APIRequestFactory, force_authenticate + from knowledge.views.version import KnowledgeVersionView + factory = APIRequestFactory() + def call(method, data=None, version_id=None): + request = getattr(factory, method)('/versions', data=data or {}, format='json') + force_authenticate(request, user=self.user, token=SimpleNamespace(role_list=['ADMIN'], permission_list=[])) + kw = {'workspace_id': 'default', 'knowledge_id': self.root.id} + if version_id: kw['version_id'] = version_id + return KnowledgeVersionView.as_view()(request, **kw) + created = call('post', {'name': '接口创建'}) + self.assertEqual(json.loads(created.content)['code'], 200) + version_id = json.loads(created.content)['data']['id'] + detail = call('get', version_id=version_id) + self.assertEqual(json.loads(detail.content)['data']['documents'][0]['name'], self.doc.name) + removed = call('delete', version_id=version_id) + self.assertEqual(json.loads(removed.content)['code'], 200) diff --git a/apps/knowledge/versions.py b/apps/knowledge/versions.py new file mode 100644 index 000000000..c877e1a48 --- /dev/null +++ b/apps/knowledge/versions.py @@ -0,0 +1,331 @@ +"""Version snapshots reuse existing retrieval storage without exposing another knowledge base.""" +import copy +import hashlib +import json +from datetime import timedelta +import uuid_utils.compat as uuid +from django.db import connection, transaction +from django.db.models import Q, OuterRef, Subquery +from django.core.exceptions import ValidationError as DjangoValidationError +from django.utils import timezone +from rest_framework.exceptions import ValidationError +from common.exception.app_exception import AppUnauthorizedFailed, NotFound404 +from knowledge.models import (Knowledge, KnowledgeVersion, KnowledgeVersionLease, Document, Paragraph, + Problem, ProblemParagraphMapping, Tag, DocumentTag, Embedding, File, TaskType, State, Status) + +SNAPSHOT_KEY = '_version_snapshot' + + +def root_knowledge(workspace, knowledge_id, lock=False): + query = Knowledge.objects.filter(pk=knowledge_id, workspace_id=workspace).exclude(meta__has_key=SNAPSHOT_KEY) + if lock: + query = query.select_for_update() + obj = query.first() + if obj is None: + raise NotFound404(404, '知识库不存在') + return obj + + +def version_data(version): + return {'id': str(version.id), 'number': version.number, 'name': version.name, + 'description': version.description, 'creator_name': version.creator_name, + 'create_time': version.create_time, 'document_count': version.document_count, + 'paragraph_count': version.paragraph_count, 'vector_count': version.vector_count, + 'search_ready': version.search_ready, 'content_hash': version.content_hash, + 'config': {key: value for key, value in version.config.items() + if not key.startswith('_') and key != 'model_identity'}} + + +def remap(value, mapping): + if isinstance(value, dict): + return {k: remap(v, mapping) for k, v in value.items()} + if isinstance(value, list): + return [remap(v, mapping) for v in value] + if isinstance(value, str): + if value in mapping: + return mapping[value] + # File links are embedded in Markdown, HTML and document metadata. + import re + return re.sub(r'(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', + lambda m: mapping.get(m[0], m[0]), value) + return value + + +def copy_rows(model, source_rows, ids, overrides=None): + objects = [] + for row in source_rows: + values = {field.attname: copy.deepcopy(getattr(row, field.attname)) for field in model._meta.concrete_fields} + for field in model._meta.concrete_fields: + key = field.attname + value = values[key] + if key.endswith('_id') or key == 'id': + values[key] = ids.get(str(value), value) + elif key in ('content', 'title', 'name', 'desc') and isinstance(value, str): + import re + values[key] = re.sub(r'(oss/file/)([0-9a-fA-F-]{36})', + lambda match: match[1] + str(ids.get(match[2], match[2])), value) + elif isinstance(value, (dict, list, str)): + values[key] = remap(value, ids) + values.update(overrides or {}) + objects.append(model(**values)) + return model.objects.bulk_create(objects, batch_size=500) + + +def _build_index(snapshot_id): + if connection.vendor != 'postgresql': + return + with connection.cursor() as cursor: + cursor.execute('SELECT vector_dims(embedding) FROM embedding WHERE knowledge_id=%s LIMIT 1', [str(snapshot_id)]) + row = cursor.fetchone() + if row: + # Identifier and predicate are generated UUIDs; dimension comes from pgvector. + name = 'embedding_hnsw_idx_' + str(snapshot_id) + cursor.execute('CREATE INDEX "' + name + '" ON embedding USING hnsw ' + '((embedding::vector(' + str(int(row[0])) + ')) vector_cosine_ops) ' + "WHERE knowledge_id='" + str(snapshot_id) + "'::uuid") + + +def create_version(workspace, knowledge_id, user, data): + name = str(data.get('name') or '').strip() + description = str(data.get('description') or '').strip() + if not name or len(name) > 128 or len(description) > 512: + raise ValidationError('版本名称须为1~128字,说明最多512字') + nested = connection.in_atomic_block + with transaction.atomic(): + if connection.vendor == 'postgresql': + with connection.cursor() as cursor: + if nested: + cursor.execute('SHOW transaction_isolation') + if cursor.fetchone()[0] not in ('repeatable read', 'serializable'): + raise ValidationError('版本创建需使用独立的一致性事务') + else: + cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ') + root = root_knowledge(workspace, knowledge_id, lock=True) + documents = list(Document.objects.filter(knowledge=root).order_by('id')) + if any(Status(doc.status)[kind] in (State.PENDING, State.STARTED) + for doc in documents for kind in (TaskType.EMBEDDING, TaskType.GENERATE_PROBLEM, TaskType.SYNC)): + raise ValidationError('文档正在处理,请完成后创建版本') + paragraphs = list(Paragraph.objects.filter(knowledge=root).order_by('id')) + problems = list(Problem.objects.filter(knowledge=root).order_by('id')) + mappings = list(ProblemParagraphMapping.objects.filter(knowledge=root)) + tags = list(Tag.objects.filter(knowledge=root)) + doc_tags = list(DocumentTag.objects.filter(document__knowledge=root)) + vectors = list(Embedding.objects.filter(knowledge=root)) + files = list(File.objects.filter(Q(source_id=str(root.id)) | + Q(source_id__in=[str(d.id) for d in documents]) | Q(meta__knowledge_id=str(root.id)))) + number = root.version_sequence + 1 + latest = KnowledgeVersion.objects.filter(knowledge=root).first() + number = max(number, latest.number + 1 if latest else 1) + version_id, snapshot_id = uuid.uuid7(), uuid.uuid7() + ids = {str(root.id): str(snapshot_id)} + for rows in (documents, paragraphs, problems, mappings, tags, doc_tags, vectors, files): + ids.update({str(row.pk): str(uuid.uuid7()) for row in rows}) + config = {'embedding_model_id': str(root.embedding_model_id) if root.embedding_model_id else None, + 'embedding_params': (root.meta or {}).get('embedding_params', {}), + 'segmentation': (root.meta or {}).get('segmentation', {}), 'knowledge_name': root.name, + 'knowledge_type': root.type, 'metadata': (root.meta or {}).get('business_metadata', {})} + model = root.embedding_model + if model: + config['model_identity'] = {'provider': model.provider, 'model_name': model.model_name, + 'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()} + content = {'documents': [[str(d.id), d.name, d.is_active, d.meta] for d in documents], + 'paragraphs': [[str(p.id), str(p.document_id), p.position, p.title, p.content, p.is_active] for p in paragraphs], + 'problems': [[str(p.id), p.content] for p in problems], + 'mappings': [[str(p.problem_id), str(p.paragraph_id)] for p in mappings], 'config': config} + digest = hashlib.sha256(json.dumps(content, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest() + config['_document_ids'] = {str(d.id): ids[str(d.id)] for d in documents} + metadata = copy.deepcopy(root.meta or {}) + metadata.pop('_version_sequence', None) + metadata.pop('parser_id', None) + metadata.update({SNAPSHOT_KEY: str(root.id), '_version_id': str(version_id)}) + snapshot = Knowledge.objects.create(id=snapshot_id, name=(root.name[:125] + ' · v' + str(number))[:150], + desc=root.desc, workspace_id=root.workspace_id, folder_id=root.folder_id, user_id=root.user_id, + embedding_model_id=root.embedding_model_id, scope=root.scope, type=0, + file_size_limit=root.file_size_limit, file_count_limit=root.file_count_limit, meta=metadata) + copy_rows(Document, documents, ids, {'knowledge_id': snapshot_id, 'type': 0}) + copy_rows(Paragraph, paragraphs, ids, {'knowledge_id': snapshot_id, 'hit_num': 0}) + copy_rows(Problem, problems, ids, {'knowledge_id': snapshot_id, 'hit_num': 0}) + copy_rows(ProblemParagraphMapping, mappings, ids, {'knowledge_id': snapshot_id}) + copy_rows(Tag, tags, ids, {'knowledge_id': snapshot_id}) + copy_rows(DocumentTag, doc_tags, ids) + # Preserve binary large-object references; only the last File reference frees the bytes. + cloned_files = copy_rows(File, files, ids, {'source_id': str(snapshot_id)}) + for file in cloned_files: + file.meta = {**(file.meta or {}), '_version_id': str(version_id), 'knowledge_id': str(root.id)} + File.objects.bulk_update(cloned_files, ['meta'], batch_size=500) + copied_vectors = copy_rows(Embedding, vectors, ids, {'knowledge_id': snapshot_id}) + for old, new in zip(vectors, copied_vectors): + new.source_id = ids.get(str(old.source_id), old.source_id) + Embedding.objects.bulk_update(copied_vectors, ['source_id'], batch_size=500) + active_docs = {d.id for d in documents if d.is_active} + active_paragraphs = {p.id for p in paragraphs if p.is_active and p.document_id in active_docs} + indexed = {v.paragraph_id for v in vectors if v.is_active} + search_ready = bool(model and active_paragraphs and active_paragraphs <= indexed) + _build_index(snapshot_id) + version = KnowledgeVersion.objects.create(id=version_id, knowledge=root, snapshot=snapshot, + number=number, name=name, description=description, creator=user, + creator_name=user.nick_name or user.username, document_count=len(documents), + paragraph_count=len(paragraphs), vector_count=len(vectors), search_ready=search_ready, + content_hash=digest, config=config) + root.version_sequence = number + root.save(update_fields=['version_sequence', 'update_time']) + return version + + +def get_version(workspace, knowledge_id, version_id, lock=False): + root_knowledge(workspace, knowledge_id) + query = KnowledgeVersion.objects.filter(knowledge_id=knowledge_id, knowledge__workspace_id=workspace, pk=version_id) + if lock: + query = query.select_for_update() + obj = query.first() + if obj is None: + raise NotFound404(404, '知识库版本不存在') + return obj + + +def validate_selection(selection, workspace, knowledge_ids): + if not isinstance(selection, dict) or len(selection) > 100: + raise ValidationError('版本选择无效') + permitted = {str(k) for k in knowledge_ids} + for root_id, chosen in selection.items(): + if str(root_id) not in permitted: + raise ValidationError('只能为已关联的知识库选择版本') + if not chosen: + continue + if chosen == 'latest': + version = KnowledgeVersion.objects.filter(knowledge_id=root_id, knowledge__workspace_id=workspace, + search_ready=True).first() + else: + try: + version = KnowledgeVersion.objects.filter(pk=chosen, knowledge_id=root_id, + knowledge__workspace_id=workspace, search_ready=True).first() + except (ValueError, DjangoValidationError): + version = None + if version is None: + raise ValidationError('请选择已就绪且属于该知识库的版本') + + +def application_selection(knowledge_setting, work_flow=None): + selection = dict((knowledge_setting or {}).get('version_selection') or {}) + knowledge_ids = [] + for node in (work_flow or {}).get('nodes', []): + if node.get('type') != 'search-knowledge-node': + continue + data = node.get('properties', {}).get('node_data', {}) + knowledge_ids.extend(data.get('knowledge_id_list') or []) + for root_id, chosen in (data.get('knowledge_setting') or {}).get('version_selection', {}).items(): + if root_id in selection and selection[root_id] != chosen: + raise ValidationError('同一工作流中的知识库须使用相同版本') + selection[root_id] = chosen + return selection, knowledge_ids + + +@transaction.atomic +def pin_selection(selection, workspace, chat_id, pinned=None): + pinned = dict(pinned or {}) + for root_id, chosen in sorted((selection or {}).items()): + if root_id in pinned or not chosen: + continue + root_knowledge(workspace, root_id, lock=True) + existing = KnowledgeVersionLease.objects.filter(chat_id=str(chat_id), version__knowledge_id=root_id, + expires_at__gt=timezone.now()).select_related('version').first() + if existing: + pinned[str(root_id)] = str(existing.version_id) + continue + query = KnowledgeVersion.objects.select_for_update().filter(knowledge_id=root_id, + knowledge__workspace_id=workspace, search_ready=True) + version = query.first() if chosen == 'latest' else query.filter(pk=chosen).first() + if version is None: + raise ValidationError('选定知识库版本不可用,请检查助手设置') + pinned[str(root_id)] = str(version.id) + for root_id, version_id in sorted(pinned.items()): + root_knowledge(workspace, root_id, lock=True) + version = KnowledgeVersion.objects.select_for_update().filter(pk=version_id, knowledge_id=root_id, + knowledge__workspace_id=workspace, search_ready=True).first() + if version is None: + raise ValidationError('会话使用的知识库版本已删除,请新建会话') + KnowledgeVersionLease.objects.update_or_create(version=version, chat_id=str(chat_id), + defaults={'expires_at': timezone.now() + timedelta(minutes=35)}) + return pinned + + +def resolve_search(knowledge_ids, pinned, workspace): + result = [] + for root_id in map(str, knowledge_ids): + if root_id not in (pinned or {}): + root_knowledge(workspace, root_id) + result.append(root_id) + continue + version = get_version(workspace, root_id, pinned[root_id]) + if not version.search_ready: + raise ValidationError('知识库版本索引不可用') + identity = version.config.get('model_identity') + model = version.snapshot.embedding_model + if not model or (identity and identity != {'provider': model.provider, 'model_name': model.model_name, + 'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}): + raise ValidationError('历史版本的向量模型配置已变更,请恢复原模型配置') + result.append(str(version.snapshot_id)) + return result + + +def resolve_document_scope(document_ids, pinned, workspace): + if document_ids is None: + return None + mapping = {} + for root_id, version_id in (pinned or {}).items(): + mapping.update(get_version(workspace, root_id, version_id).config.get('_document_ids', {})) + return [mapping.get(str(document_id), str(document_id)) for document_id in document_ids] + + +@transaction.atomic +def delete_version(workspace, knowledge_id, version_id): + root = root_knowledge(workspace, knowledge_id, lock=True) + version = get_version(workspace, knowledge_id, version_id, lock=True) + from application.models import Application, ApplicationVersion + key = 'knowledge_setting__version_selection__' + str(root.id) + is_latest = KnowledgeVersion.objects.filter(knowledge=root, search_ready=True).first() + # Public chat uses only the latest publication. Superseded application history + # must not prevent users from retiring an otherwise unused knowledge version. + publications = ApplicationVersion.objects.filter(pk=Subquery(ApplicationVersion.objects.filter( + application_id=OuterRef('application_id')).order_by('-create_time', '-id').values('pk')[:1])) + for query in (Application.objects.all(), publications): + refs = Q(**{key: str(version.id)}) + if is_latest and is_latest.id == version.id: + refs |= Q(**{key: 'latest'}) + if query.filter(refs).exists(): + raise ValidationError('版本已被助手或已发布配置使用,请先更换版本并重新发布') + for flow in query.filter(type='WORK_FLOW').values_list('work_flow', flat=True).iterator(): + for node in (flow or {}).get('nodes', []): + chosen = ((node.get('properties', {}).get('node_data', {}).get('knowledge_setting') or {}) + .get('version_selection') or {}).get(str(root.id)) + if chosen == str(version.id) or (chosen == 'latest' and is_latest and is_latest.id == version.id): + raise ValidationError('版本已被工作流或已发布配置使用,请先更换版本并重新发布') + KnowledgeVersionLease.objects.filter(version=version, expires_at__lte=timezone.now()).delete() + if version.leases.exists(): + raise ValidationError('版本有活跃会话,请会话结束后再删除') + snapshot_id = version.snapshot_id + Embedding.objects.filter(knowledge_id=snapshot_id).delete() + ProblemParagraphMapping.objects.filter(knowledge_id=snapshot_id).delete() + DocumentTag.objects.filter(document__knowledge_id=snapshot_id).delete() + Tag.objects.filter(knowledge_id=snapshot_id).delete() + Paragraph.objects.filter(knowledge_id=snapshot_id).delete() + Document.objects.filter(knowledge_id=snapshot_id).delete() + Problem.objects.filter(knowledge_id=snapshot_id).delete() + File.objects.filter(source_id=str(snapshot_id)).delete() + if connection.vendor == 'postgresql': + with connection.cursor() as cursor: + cursor.execute('DROP INDEX IF EXISTS "embedding_hnsw_idx_' + str(snapshot_id) + '"') + version.delete() + Knowledge.objects.filter(pk=snapshot_id).delete() + + +def guard_snapshot_request(request): + match = getattr(request, 'resolver_match', None) + kwargs = match.kwargs if match else {} + knowledge_ids = [kwargs[key] for key in ('knowledge_id', 'target_knowledge_id') if kwargs.get(key)] + if knowledge_ids and Knowledge.objects.filter(pk__in=knowledge_ids, meta__has_key=SNAPSHOT_KEY).exists(): + raise AppUnauthorizedFailed(403, '历史版本只读,请通过原知识库的版本管理查看') + file_id = kwargs.get('file_id') + if file_id and request.method not in ('GET', 'HEAD', 'OPTIONS') and File.objects.filter( + pk=file_id, meta__has_key='_version_id').exists(): + raise AppUnauthorizedFailed(403, '历史版本文件不能单独删除') diff --git a/apps/knowledge/views/version.py b/apps/knowledge/views/version.py new file mode 100644 index 000000000..3120c427f --- /dev/null +++ b/apps/knowledge/views/version.py @@ -0,0 +1,55 @@ +from rest_framework.views import APIView +from rest_framework.exceptions import ValidationError +from common.auth import TokenAuth +from common.auth.authentication import has_permissions +from common.constants.permission_constants import PermissionConstants as P, RoleConstants as R, ViewPermission, CompareConstants +from common.log.log import log +from common.result import result +from knowledge import versions +from knowledge.models import KnowledgeVersion, Document, Paragraph + + +def rights(action): + permission = P.KNOWLEDGE_READ if action == 'read' else P.KNOWLEDGE_EDIT + return (R.ADMIN, permission.get_workspace_knowledge_permission(), + permission.get_workspace_permission_workspace_manage_role(), R.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([R.USER.get_workspace_role()], [P.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND)) + + +class KnowledgeVersionView(APIView): + authentication_classes = [TokenAuth] + + @has_permissions(*rights('read')) + def get(self, request, workspace_id, knowledge_id, version_id=None): + versions.root_knowledge(workspace_id, knowledge_id) + if not version_id: + return result.success([versions.version_data(v) for v in KnowledgeVersion.objects.filter(knowledge_id=knowledge_id)]) + version = versions.get_version(workspace_id, knowledge_id, version_id) + if request.query_params.get('content') == 'paragraphs': + try: + page = max(1, int(request.query_params.get('page', 1))) + size = max(1, min(100, int(request.query_params.get('page_size', 20)))) + except (ValueError, TypeError): + raise ValidationError('分页参数无效') + query = Paragraph.objects.filter(knowledge_id=version.snapshot_id) + document_id = request.query_params.get('document_id') + if document_id: + query = query.filter(document_id=document_id) + search = request.query_params.get('search', '').strip() + if search: + query = query.filter(content__icontains=search) + return result.success({'total': query.count(), 'records': list(query.order_by('document_id', 'position', 'id') + .values('id', 'document_id', 'document__name', 'title', 'content', 'position', 'is_active')[(page-1)*size:page*size])}) + return result.success({**versions.version_data(version), 'documents': list(Document.objects.filter( + knowledge_id=version.snapshot_id).order_by('name', 'id').values('id', 'name', 'char_length', 'is_active'))}) + + @has_permissions(*rights('edit')) + @log(menu='Knowledge Base', operate='Create knowledge version') + def post(self, request, workspace_id, knowledge_id): + return result.success(versions.version_data(versions.create_version(workspace_id, knowledge_id, request.user, request.data))) + + @has_permissions(*rights('edit')) + @log(menu='Knowledge Base', operate='Delete knowledge version') + def delete(self, request, workspace_id, knowledge_id, version_id): + versions.delete_version(workspace_id, knowledge_id, version_id) + return result.success(True) diff --git a/apps/models_provider/context_policy.py b/apps/models_provider/context_policy.py new file mode 100644 index 000000000..d7244959c --- /dev/null +++ b/apps/models_provider/context_policy.py @@ -0,0 +1,34 @@ +"""Deployment defaults; explicit model settings always take precedence.""" + +DEFAULT_CONTEXT_TOKENS = 65536 +DEFAULT_SUMMARY_OUTPUT_TOKENS = 8192 +DEFAULT_SAFETY_TOKENS = 4096 + + +def model_context(params): + value = params.get('context_tokens') + if value is None: + value = params.get('num_ctx', DEFAULT_CONTEXT_TOKENS) + if isinstance(value, str) and value.isdigit(): + value = int(value) + if type(value) is not int or not 1024 <= value <= 2000000: + raise ValueError('模型上下文须为 1024 至 2000000 的整数') + return value + + +def summary_defaults(params): + result = dict(params) + context = model_context(result) + result['context_tokens'] = context + result.setdefault('safety_tokens', min(DEFAULT_SAFETY_TOKENS, max(256, context // 16))) + safety = int(result['safety_tokens']) + output = result.get('output_tokens', result.get('max_tokens')) + if output is None: + output = min(DEFAULT_SUMMARY_OUTPUT_TOKENS, max(256, context // 8), + int(result.get('max_output_tokens') or DEFAULT_SUMMARY_OUTPUT_TOKENS)) + result['output_tokens'] = int(output) + result.setdefault('max_output_tokens', max(int(output), min(DEFAULT_SUMMARY_OUTPUT_TOKENS, + context - safety - 1024))) + result.setdefault('max_input_tokens', context) + result.setdefault('input_tokens', min(int(result['max_input_tokens']), context - int(output) - safety)) + return result diff --git a/apps/models_provider/test_context_policy.py b/apps/models_provider/test_context_policy.py new file mode 100644 index 000000000..9bc6a217d --- /dev/null +++ b/apps/models_provider/test_context_policy.py @@ -0,0 +1,74 @@ +"""Context inheritance and native provider options; no network or database.""" +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import patch + +from models_provider.context_policy import model_context, summary_defaults + + +class ContextPolicyTests(TestCase): + def test_platform_budget_is_64k_with_room_for_output_and_prompt(self): + from system_manage.ai_hub_organize import resolve_settings + defaults = summary_defaults({}) + self.assertEqual(defaults, {'context_tokens': 65536, 'max_input_tokens': 65536, + 'input_tokens': 53248, 'max_output_tokens': 8192, 'output_tokens': 8192, 'safety_tokens': 4096}) + self.assertEqual(resolve_settings({}, {}), + {'context_tokens': 65536, 'output_tokens': 8192, 'safety_tokens': 4096}) + + def test_explicit_model_limits_and_summary_budgets_win(self): + custom = {'context_tokens': '1000000', 'max_input_tokens': '1000000', 'input_tokens': '131072', + 'max_output_tokens': '384000', 'output_tokens': '16384', 'safety_tokens': '4096'} + defaults = summary_defaults(custom) + self.assertEqual(defaults['context_tokens'], 1000000) + for key in custom.keys() - {'context_tokens', 'output_tokens'}: + self.assertEqual(defaults[key], custom[key]) + self.assertEqual(defaults['output_tokens'], 16384) + self.assertEqual(model_context({'num_ctx': '131072'}), 131072) + + def test_smaller_model_and_maximum_output_are_respected(self): + small = summary_defaults({'context_tokens': 4096, 'max_output_tokens': 256}) + self.assertEqual(small['output_tokens'], 256) + self.assertLess(small['input_tokens'] + small['output_tokens'], 4096) + for value in (True, 0, -1, 1023, 2000001, 'invalid'): + with self.subTest(value=value), self.assertRaises(ValueError): + model_context({'context_tokens': value}) + + def test_only_llm_gets_default_context_from_model_configuration(self): + from models_provider.tools import get_model_default_params + model = SimpleNamespace(model_type='LLM', model_params_form=[]) + self.assertEqual(get_model_default_params(model), {'context_tokens': 65536}) + model.model_params_form = [{'field': 'num_ctx', 'default_value': '131072'}] + self.assertEqual(get_model_default_params(model)['context_tokens'], 131072) + model.model_type = 'EMBEDDING' + model.model_params_form = [] + self.assertEqual(get_model_default_params(model), {}) + + def test_ollama_native_context_defaults_and_explicit_override(self): + from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel + for params, expected in (({}, 65536), ({'context_tokens': 131072}, 131072), + ({'num_ctx': 32768}, 32768), + ({'context_tokens': 131072, 'num_ctx': 32768}, 131072)): + with patch.object(OllamaChatModel, '__new__', return_value=SimpleNamespace()) as constructor: + OllamaChatModel.new_instance('LLM', 'local', {'api_base': 'http://localhost:11434'}, **params) + self.assertEqual(constructor.call_args.kwargs['num_ctx'], expected) + self.assertNotIn('context_tokens', constructor.call_args.kwargs) + + def test_native_application_override_is_not_hidden_by_platform_default(self): + from models_provider import tools + model = SimpleNamespace(model_type='LLM', model_params_form=[]) + with patch.object(tools, 'get_model_by_id', return_value=model), \ + patch.object(tools.ModelManage, 'get_model', side_effect=lambda key, factory: factory(key)), \ + patch.object(tools, 'get_model') as create: + tools.get_model_instance_by_model_workspace_id('test', 'default', num_ctx=131072) + self.assertEqual(create.call_args.kwargs['context_tokens'], 131072) + + def test_metadata_and_preview_share_effective_budget_without_model_calls(self): + from system_manage.mcp_business import summary_model_metadata + from models_provider.tools import get_model_default_params + model = SimpleNamespace(id='test', name='test', model_type='LLM', provider='model_ollama_provider', + model_params_form=[]) + with patch('models_provider.tools.get_model') as invoke: + metadata = summary_model_metadata(model) + invoke.assert_not_called() + self.assertEqual(metadata['summary_defaults']['input_tokens'], 53248) + self.assertEqual(metadata['summary_limits']['context_tokens'], 65536) diff --git a/apps/system_manage/ai_hub_service_api.py b/apps/system_manage/ai_hub_service_api.py new file mode 100644 index 000000000..92e04bba4 --- /dev/null +++ b/apps/system_manage/ai_hub_service_api.py @@ -0,0 +1,104 @@ +import copy +import json +import re + +from django.db import transaction +from rest_framework.exceptions import ValidationError +from system_manage.models import SystemSetting + +SETTING = 11029 +DEFAULT = {'method': 'POST', 'parameters': [ + {'name': 'markdown', 'target': 'markdown', 'type': 'string', 'required': False}, + {'name': 'name', 'target': 'name', 'type': 'string', 'required': False}, + {'name': 'file', 'target': 'file', 'type': 'file', 'required': False}, +]} +NAME = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,63}$') + + +def validate(config): + if not isinstance(config, dict) or set(config) != {'method', 'parameters'} or config['method'] not in ('GET', 'POST') or not isinstance(config['parameters'], list): + raise ValidationError('服务接口配置无效') + names, targets = set(), set() + for p in config['parameters']: + if not isinstance(p, dict) or set(p) != {'name', 'target', 'type', 'required'}: + raise ValidationError('服务参数配置无效') + name, target = p['name'], p['target'] + if not isinstance(name, str) or not isinstance(target, str) or not NAME.fullmatch(name) or name.lower() in ('data', '__proto__', 'constructor', 'prototype') or name in names or target in targets: + raise ValidationError('服务参数名称重复或无效') + if target not in ('markdown', 'name', 'file') or p['type'] != ('file' if target == 'file' else 'string') or not isinstance(p['required'], bool): + raise ValidationError('服务参数类型无效') + if config['method'] == 'GET' and target == 'file': + raise ValidationError('GET 不支持文件参数') + names.add(name); targets.add(target) + return copy.deepcopy(config) + + +def get_config(service_id): + meta = SystemSetting.objects.filter(pk=SETTING).values_list('meta', flat=True).first() or {} + return copy.deepcopy(meta.get('services', {}).get(str(service_id), DEFAULT)) + + +def save_config(service_id, config=None): + with transaction.atomic(): + row, _ = SystemSetting.objects.get_or_create(type=SETTING, defaults={'meta': {'services': {}}}) + row = SystemSetting.objects.select_for_update().get(pk=row.pk) + meta = dict(row.meta or {}); services = dict(meta.get('services', {})) + if config is None: + services.pop(str(service_id), None) + else: + services[str(service_id)] = validate(config) + meta['services'] = services; row.meta = meta; row.save(update_fields=['meta']) + + +def parse_request(request, config): + if request.method != config['method']: + raise ValidationError('请求方法与服务配置不一致') + if request.method == 'GET': + if len(request.META.get('QUERY_STRING', '')) > 8192: + raise ValidationError('请求参数过长') + source = request.query_params + elif 'data' in request.data: + if hasattr(request.data, 'getlist') and len(request.data.getlist('data')) != 1: + raise ValidationError('参数不能重复') + try: + source = json.loads(request.data['data']) + except (ValueError, TypeError): + raise ValidationError('任务参数无效') + if not isinstance(source, dict) or set(request.data) - {'data', *request.FILES.keys()}: + raise ValidationError('任务参数无效') + else: + source = request.data + if not isinstance(source, dict): + raise ValidationError('任务参数无效') + params = config['parameters']; names = {p['name'] for p in params} + legacy_priority = request.method == 'POST' and config == DEFAULT + if legacy_priority: + names.add('priority') + if set(source) - names or set(request.FILES) - names: + raise ValidationError('包含未知参数') + output = {}; upload = None + if legacy_priority and 'priority' in source: + priority = source.get('priority') + if hasattr(source, 'getlist') and len(source.getlist('priority')) > 1: + raise ValidationError('参数不能重复') + if isinstance(priority, bool) or not isinstance(priority, int) or not 0 <= priority <= 9: + raise ValidationError('优先级无效') + output['priority'] = priority + for p in params: + name = p['name'] + if hasattr(request.FILES, 'getlist') and len(request.FILES.getlist(name)) > 1: + raise ValidationError('参数不能重复') + if hasattr(source, 'getlist') and len(source.getlist(name)) > 1: + raise ValidationError('参数不能重复') + value = request.FILES.get(name) if p['type'] == 'file' else source.get(name) + if p['type'] == 'file' and name in source and name not in request.FILES: + raise ValidationError('文件参数无效') + if p['type'] != 'file' and name in request.FILES: + raise ValidationError('参数类型无效') + if value is None or value == '': + if p['required']: raise ValidationError('缺少必填参数') + continue + if p['type'] == 'file': upload = value + elif not isinstance(value, str): raise ValidationError('参数类型无效') + else: output[p['target']] = value + return output, upload diff --git a/apps/system_manage/artifact_settings.py b/apps/system_manage/artifact_settings.py new file mode 100644 index 000000000..84e6af071 --- /dev/null +++ b/apps/system_manage/artifact_settings.py @@ -0,0 +1,26 @@ +"""Administrator-controlled limit for saved mind maps.""" +from django.db import transaction +from rest_framework.exceptions import ValidationError +from system_manage.models import SystemSetting + +SETTING = 11031 +DEFAULT_LIMIT = 10 + + +def limit(): + row = SystemSetting.objects.filter(pk=SETTING).first() + value = (row.meta or {}).get('limit', DEFAULT_LIMIT) if row else DEFAULT_LIMIT + return value if type(value) is int and 1 <= value <= 10000 else DEFAULT_LIMIT + + +@transaction.atomic +def save(user, data): + from system_manage.ai_hub import require_admin + require_admin(user) + if set(data) != {'limit'} or type(data['limit']) is not int or not 1 <= data['limit'] <= 10000: + raise ValidationError('保存条数须为1至10000的整数') + row, _ = SystemSetting.objects.get_or_create(type=SETTING, defaults={'meta': {}}) + row = SystemSetting.objects.select_for_update().get(pk=row.pk) + row.meta = {'limit': data['limit']} + row.save(update_fields=['meta', 'update_time']) + return {'limit': data['limit']} diff --git a/apps/system_manage/heading_service.py b/apps/system_manage/heading_service.py new file mode 100644 index 000000000..16602dc25 --- /dev/null +++ b/apps/system_manage/heading_service.py @@ -0,0 +1,70 @@ +"""Authorized workbench entry for outline inspection, rules, manual edits and model jobs.""" +from rest_framework.exceptions import ValidationError +from system_manage import ai_hub as hub +from system_manage.heading_structure import analyze, artifact_state, manual_edit, validate_manual + + +def invoke_model(workspace, user, node, markdown, source, parent, progress): + """Optional title assistance must not discard successful document parsing.""" + from system_manage import mcp_runtime + output, report = analyze(markdown, source, source.get('heading_structure')) + if not report['uncertain_count']: + report['model'] = {'status': 'skipped', 'reason': '现有依据充分,无需模型校正'} + return {'text': output, 'structured': {'heading_structure': report}} + try: + model = hub.model_for(workspace, user, node['model_id']) + if model.update_time.isoformat() != node['resource_version']: + raise ValueError('model changed') + params = {'text': markdown, 'operation': 'heading_repair', + 'heading_structure': source.get('heading_structure', {}), 'heading_source': source, + 'temperature': 0, **{key: node[key] for key in ('context_tokens', 'safety_tokens') if key in node}} + if node.get('output_tokens') is not None: + params['max_tokens'] = node['output_tokens'] + return mcp_runtime.invoke(workspace, user, 'model:' + str(model.id), params, + parent=parent, timeout=node.get('timeout', 180), progress=progress, scope={'source': 'mindmap'}) + except Exception: + output, report = analyze(markdown, source, source.get('heading_structure')) + report['model'] = {'status': 'fallback', 'reason': '模型暂不可用,已保留解析和规则结果'} + return {'text': output, 'structured': {'heading_structure': report}} + + +def process(workspace, user, data, request=None): + markdown = hub.text(data.get('markdown'), '文档内容') + source, previous, original = {}, {}, markdown + if data.get('artifact_id'): + artifact = hub.one('artifacts', data['artifact_id'], workspace, user, edit=True) + previous, original = artifact_state(artifact), artifact.original + if artifact.task_id: + source = artifact.task.result.get('source') or {} + elif data.get('task_id'): + task = hub.one('tasks', data['task_id'], workspace, user) + if task.status != 'SUCCESS': + raise ValidationError('解析任务尚未完成') + source = task.result.get('source') or {} + previous, original = source.get('heading_structure', {}), task.result.get('original', markdown) + incoming = data.get('heading_structure') + if incoming is not None and not isinstance(incoming, dict): + raise ValidationError('层级数据无效') + mode = data.get('mode', 'inspect') + if mode not in ('inspect', 'rules', 'manual', 'model'): + raise ValidationError('层级操作无效') + try: + # Current line positions win; IDs/evidence/manual state come from the prior report. + _, report = analyze(markdown, source, previous or incoming, repair=False) + if incoming is not None: + report = validate_manual(report, incoming) + output, report = analyze(markdown, source, report, repair=mode in ('rules', 'model')) + if mode == 'manual': + action = data.get('action') + if not isinstance(action, dict): + raise ValueError('请选择层级操作') + output, report = manual_edit(output, report, action) + except (ValueError, TypeError) as exc: + raise ValidationError(str(exc) if isinstance(exc, ValueError) else '层级数据无效') from None + if mode != 'model': + return {'markdown': output, 'heading_structure': report, 'original': original} + task = hub.submit(workspace, user, {'name': '标题层级校正', 'markdown': output, + 'original': original, 'heading_structure': report, '_heading_source': source, + 'nodes': [{'type': 'organize', 'operation': 'heading_repair', 'model_id': data.get('model_id'), + 'temperature': 0, 'prompt': '仅校正标题层级,保留正文。'}, {'type': 'render'}]}, request=request) + return {'task_id': str(task.id), 'status': task.status} diff --git a/apps/system_manage/heading_structure.py b/apps/system_manage/heading_structure.py new file mode 100644 index 000000000..40f5539b9 --- /dev/null +++ b/apps/system_manage/heading_structure.py @@ -0,0 +1,456 @@ +"""Conservative outline repair. Rules/models change heading markers, never body text. + +All line numbers are zero based. Stable IDs and explicit manual locks survive +repair and artifact revisions. Layout scores are heuristics, not probabilities. +""" +import copy +import hashlib +import json +import math +import re +from collections import Counter, defaultdict +from statistics import median + +MAX_HEADINGS = 6000 +HEADING = re.compile(r'^( {0,3})(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$') +DECIMAL = re.compile(r'^(\d{1,3}(?:\.\d{1,3}){0,5})(?:[.、.])?[ \t]+(\S.*)$') +CHINESE = [('chapter', re.compile(r'^第[一二三四五六七八九十百千零〇\d]+[章节篇部]\s*\S*')), + ('chinese', re.compile(r'^[一二三四五六七八九十百]+[、..]\s*\S+')), + ('paren_chinese', re.compile(r'^[((][一二三四五六七八九十百]+[))]\s*\S+')), + ('paren_digit', re.compile(r'^[((]\d{1,3}[))]\s*\S+'))] + + +def label(text): + return re.sub(r'\s+', ' ', re.sub(r'[*_`]', '', text)).strip() + + +def _short(text): + return 0 < len(text) <= 120 and not re.search(r'[。!?;;!?]|https?://|\|', text) + + +def _number(text): + match = DECIMAL.match(text) + if not match or not _short(text) or re.match(r'^(?:19|20)\d{2}\b', text): + return None + prefix = match[1] + if len(prefix.split('.')) == 4 and all(int(x) <= 255 for x in prefix.split('.')) and ( + int(prefix.split('.')[0]) >= 10 or re.search(r'\b(?:IP|gateway|address|DNS)\b|地址|网关|掩码', match[2], re.I)): + # An IP address is not a chapter number without independent evidence. + return None + if re.match(r'^(?:版本|version|v\d|\d+(?:\.\d+)+\s*(?:版|版本))', text, re.I): + return None + return prefix + + +def _protected_lines(markdown): + from markdown_it import MarkdownIt + tokens = MarkdownIt().enable('table').parse(markdown) + if len(tokens) > 60000: + raise ValueError('文档结构过于复杂,请拆分文档') + excluded, lists = set(), set() + depth = 0 + for token in tokens: + if token.type in ('bullet_list_open', 'ordered_list_open'): + depth += 1 + if token.map and token.type in ('fence', 'code_block', 'table_open', 'html_block'): + excluded.update(range(*token.map)) + if token.map and depth: + lists.update(range(*token.map)) + if token.type in ('bullet_list_close', 'ordered_list_close'): + depth -= 1 + return excluded, lists + + +def _evidence(source): + """Normalize native evidence and MinerU middle JSON without exposing provider config.""" + evidence = [] + for item in source.get('blocks', []) if isinstance(source, dict) else []: + if isinstance(item, dict) and isinstance(item.get('text'), str): + evidence.append(dict(item)) + middle = source.get('middle_json', {}) if isinstance(source, dict) else {} + for page in middle.get('pdf_info', []) if isinstance(middle, dict) else []: + if not isinstance(page, dict): + continue + size = page.get('page_size') or [0, 0] + for block in (page.get('para_blocks') or []) + (page.get('discarded_blocks') or []): + if not isinstance(block, dict): + continue + text = ''.join(str(s.get('content', '')) for line in block.get('lines', []) + for s in line.get('spans', []) if s.get('type') == 'text') + evidence.append({'text': text, 'page': page.get('page_idx', 0), 'bbox': block.get('bbox'), + 'page_height': size[1] if len(size) > 1 else 0, 'kind': block.get('type'), + 'height': block.get('line_avg_height') or median([ + abs(line['bbox'][3] - line['bbox'][1]) for line in block.get('lines', []) + if isinstance(line.get('bbox'), list) and len(line['bbox']) == 4] or [0])}) + for item in evidence: + for key in ('font_size', 'height', 'page_height'): + value = item.get(key) + item[key] = value if type(value) in (int, float) and math.isfinite(value) and 0 <= value < 100000 else 0 + box = item.get('bbox') + if not isinstance(box, list) or len(box) != 4 or any(type(x) not in (int, float) or not math.isfinite(x) for x in box): + item['bbox'] = None + return evidence[:40000] + + +def analyze(markdown, source=None, previous=None, repair=True): + source, previous = source or {}, previous or {} + if not isinstance(source, dict) or not isinstance(previous, dict): + raise ValueError('标题结构数据无效') + old_headings = previous.get('headings', []) + if not isinstance(old_headings, list) or len(old_headings) > MAX_HEADINGS or any( + not isinstance(h, dict) or not isinstance(h.get('id'), str) or len(h['id']) > 128 or + not isinstance(h.get('text'), str) or type(h.get('level')) is not int or not 0 <= h['level'] <= 6 + for h in old_headings): + raise ValueError('标题候选数据无效') + lines = markdown.splitlines(keepends=True) + excluded, list_lines = _protected_lines(markdown) + evidence = _evidence(source) + if not evidence: + evidence = _evidence({'blocks': [{**r, 'kind': 'title' if r.get('level') else 'text'} for r in old_headings]}) + by_text = defaultdict(list) + bookmarks = defaultdict(list) + for item in source.get('bookmarks', []): + if isinstance(item, dict) and type(item.get('level')) is int and 1 <= item['level'] <= 6: + bookmarks[label(item.get('text', ''))].append(item) + for block in evidence: + by_text[label(block['text'])].append(block) + old_by_text = defaultdict(list) + for item in previous.get('headings', []): + if isinstance(item, dict): + old_by_text[item.get('text', '')].append(item) + used_ids, counts, records = set(), Counter(), [] + body_sizes = [b.get('font_size', 0) or b.get('height', 0) for b in evidence + if b.get('kind') in ('text', 'paragraph') and (b.get('font_size', 0) or b.get('height', 0)) > 0] + body_size = median(body_sizes) if body_sizes else 0 + evidence_cursor = Counter() + margins = defaultdict(set) + for b in evidence: + box, height = b.get('bbox'), b.get('page_height', 0) + if isinstance(box, list) and len(box) == 4 and height and (box[3] <= height * .1 or box[1] >= height * .9): + margins[label(b['text'])].add(b.get('page')) + toc_lines = set() + for pos, raw in enumerate(lines): + if label(re.sub(r'^#{1,6}\s+', '', raw.strip())).lower() not in ('目录', 'contents', 'table of contents'): + continue + candidates = [] + for index in range(pos + 1, min(len(lines), pos + 250)): + if re.search(r'(?:\.{3,}|…{2,}|·{3,})\s*\d+\s*$', lines[index]): + candidates.append(index) + elif candidates and HEADING.match(lines[index].rstrip('\r\n')): + break + if len(candidates) >= 2: + toc_lines.update(candidates) + for index, raw in enumerate(lines): + if index in excluded: + continue + line = raw.rstrip('\r\n') + match = HEADING.match(line) + text = label(match[3] if match else line.strip()) + if not text or text.startswith(('![', '